#install.packages('TDAmapper')
library(TDAmapper)
library(cluster)
library(vip)
## 
## Attaching package: 'vip'
## The following object is masked from 'package:utils':
## 
##     vi
#install.packages('kernlab’)
library(kernlab)
#install.packages(‘class’)
library(class)
#install.packages('nnet')
library(nnet)
#install.packages(‘randomForest’)
library(randomForest)
## randomForest 4.7-1.1
## Type rfNews() to see new features/changes/bug fixes.
#install.packages('e1071')
library(e1071)                                                  
#install.packages("BayesFactor")
library(BayesFactor)
## Loading required package: coda
## 
## Attaching package: 'coda'
## The following object is masked from 'package:kernlab':
## 
##     nvar
## Loading required package: Matrix
## ************
## Welcome to BayesFactor 0.9.12-4.5. If you have questions, please contact Richard Morey (richarddmorey@gmail.com).
## 
## Type BFManual() to open the manual.
## ************
library(BayesPPD)
library(bayestestR)
#install.packages('igraph')
library('igraph')
## Warning: package 'igraph' was built under R version 4.3.3
## 
## Attaching package: 'igraph'
## The following object is masked from 'package:BayesFactor':
## 
##     compare
## The following object is masked from 'package:class':
## 
##     knn
## The following objects are masked from 'package:stats':
## 
##     decompose, spectrum
## The following object is masked from 'package:base':
## 
##     union
#install.packages('locfit')
library(locfit)
## locfit 1.5-9.8    2023-06-11
#install.packages('ggplot2’)
library(ggplot2)
## 
## Attaching package: 'ggplot2'
## The following object is masked from 'package:randomForest':
## 
##     margin
## The following object is masked from 'package:kernlab':
## 
##     alpha
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:igraph':
## 
##     as_data_frame, groups, union
## The following object is masked from 'package:randomForest':
## 
##     combine
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
#install.packages('networkD3')
library(networkD3)
library(rstanarm)
## Loading required package: Rcpp
## This is rstanarm version 2.26.1
## - See https://mc-stan.org/rstanarm/articles/priors for changes to default priors!
## - Default priors may change, so it's safest to specify priors, even if equivalent to the defaults.
## - For execution on a local, multicore CPU with excess RAM we recommend calling
##   options(mc.cores = parallel::detectCores())
library(see)
#install.packages('tidyverse')
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ forcats   1.0.0     ✔ stringr   1.5.1
## ✔ lubridate 1.9.3     ✔ tibble    3.2.1
## ✔ purrr     1.0.2     ✔ tidyr     1.3.0
## ✔ readr     2.1.4
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ lubridate::%--%()       masks igraph::%--%()
## ✖ ggplot2::alpha()        masks kernlab::alpha()
## ✖ tibble::as_data_frame() masks dplyr::as_data_frame(), igraph::as_data_frame()
## ✖ dplyr::combine()        masks randomForest::combine()
## ✖ purrr::compose()        masks igraph::compose()
## ✖ purrr::cross()          masks kernlab::cross()
## ✖ tidyr::crossing()       masks igraph::crossing()
## ✖ tidyr::expand()         masks Matrix::expand()
## ✖ dplyr::filter()         masks stats::filter()
## ✖ dplyr::lag()            masks stats::lag()
## ✖ ggplot2::margin()       masks randomForest::margin()
## ✖ purrr::none()           masks locfit::none()
## ✖ tidyr::pack()           masks Matrix::pack()
## ✖ purrr::simplify()       masks igraph::simplify()
## ✖ tidyr::unpack()         masks Matrix::unpack()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
#install.packages('caret')
library(caret)
## Loading required package: lattice
## 
## Attaching package: 'caret'
## 
## The following object is masked from 'package:purrr':
## 
##     lift
## 
## The following objects are masked from 'package:rstanarm':
## 
##     compare_models, R2
#install.packages('ISLR')
library(ISLR)
#install.packages('MCMCpack')
library(MCMCpack)
## Loading required package: MASS
## 
## Attaching package: 'MASS'
## 
## The following object is masked from 'package:dplyr':
## 
##     select
## 
## ##
## ## Markov Chain Monte Carlo Package (MCMCpack)
## ## Copyright (C) 2003-2025 Andrew D. Martin, Kevin M. Quinn, and Jong Hee Park
## ##
## ## Support provided by the U.S. National Science Foundation
## ## (Grants SES-0350646 and SES-0350613)
## ##
#linstall.packages("caret")
library(caret)
library(TDA)
## 
## Attaching package: 'TDA'
## 
## The following object is masked from 'package:cluster':
## 
##     silhouette
library(TDAstats)
library(ks)
## 
## Attaching package: 'ks'
## 
## The following object is masked from 'package:TDA':
## 
##     kde
## 
## The following object is masked from 'package:MCMCpack':
## 
##     vech
## 
## The following object is masked from 'package:igraph':
## 
##     compare
## 
## The following object is masked from 'package:BayesFactor':
## 
##     compare
#install.packages('MLmetrics')
library(MLmetrics)
## 
## Attaching package: 'MLmetrics'
## 
## The following objects are masked from 'package:caret':
## 
##     MAE, RMSE
## 
## The following object is masked from 'package:base':
## 
##     Recall
#install.packages('googledrive')
library(googledrive)
#install.packages('stringr')
library(stringr)
#install.packages('ks')
library(ks)
library(GGally)
## Registered S3 method overwritten by 'GGally':
##   method from   
##   +.gg   ggplot2
#import DryBean dataset from UCI repository stored on my desktop

#Dry_Bean_Dataset **
library(readxl)
Dry_Bean_Dataset <- read_excel("~/Desktop/NCU/DissertationDatasets/DryBeanDataset/Dry_Bean_Dataset.xlsx")
  head(str(Dry_Bean_Dataset))
## tibble [13,611 × 17] (S3: tbl_df/tbl/data.frame)
##  $ Area           : num [1:13611] 28395 28734 29380 30008 30140 ...
##  $ Perimeter      : num [1:13611] 610 638 624 646 620 ...
##  $ MajorAxisLength: num [1:13611] 208 201 213 211 202 ...
##  $ MinorAxisLength: num [1:13611] 174 183 176 183 190 ...
##  $ AspectRation   : num [1:13611] 1.2 1.1 1.21 1.15 1.06 ...
##  $ Eccentricity   : num [1:13611] 0.55 0.412 0.563 0.499 0.334 ...
##  $ ConvexArea     : num [1:13611] 28715 29172 29690 30724 30417 ...
##  $ EquivDiameter  : num [1:13611] 190 191 193 195 196 ...
##  $ Extent         : num [1:13611] 0.764 0.784 0.778 0.783 0.773 ...
##  $ Solidity       : num [1:13611] 0.989 0.985 0.99 0.977 0.991 ...
##  $ roundness      : num [1:13611] 0.958 0.887 0.948 0.904 0.985 ...
##  $ Compactness    : num [1:13611] 0.913 0.954 0.909 0.928 0.971 ...
##  $ ShapeFactor1   : num [1:13611] 0.00733 0.00698 0.00724 0.00702 0.0067 ...
##  $ ShapeFactor2   : num [1:13611] 0.00315 0.00356 0.00305 0.00321 0.00366 ...
##  $ ShapeFactor3   : num [1:13611] 0.834 0.91 0.826 0.862 0.942 ...
##  $ ShapeFactor4   : num [1:13611] 0.999 0.998 0.999 0.994 0.999 ...
##  $ Class          : chr [1:13611] "SEKER" "SEKER" "SEKER" "SEKER" ...
## NULL
  summary(Dry_Bean_Dataset)
##       Area          Perimeter      MajorAxisLength MinorAxisLength
##  Min.   : 20420   Min.   : 524.7   Min.   :183.6   Min.   :122.5  
##  1st Qu.: 36328   1st Qu.: 703.5   1st Qu.:253.3   1st Qu.:175.8  
##  Median : 44652   Median : 794.9   Median :296.9   Median :192.4  
##  Mean   : 53048   Mean   : 855.3   Mean   :320.1   Mean   :202.3  
##  3rd Qu.: 61332   3rd Qu.: 977.2   3rd Qu.:376.5   3rd Qu.:217.0  
##  Max.   :254616   Max.   :1985.4   Max.   :738.9   Max.   :460.2  
##   AspectRation    Eccentricity      ConvexArea     EquivDiameter  
##  Min.   :1.025   Min.   :0.2190   Min.   : 20684   Min.   :161.2  
##  1st Qu.:1.432   1st Qu.:0.7159   1st Qu.: 36714   1st Qu.:215.1  
##  Median :1.551   Median :0.7644   Median : 45178   Median :238.4  
##  Mean   :1.583   Mean   :0.7509   Mean   : 53768   Mean   :253.1  
##  3rd Qu.:1.707   3rd Qu.:0.8105   3rd Qu.: 62294   3rd Qu.:279.4  
##  Max.   :2.430   Max.   :0.9114   Max.   :263261   Max.   :569.4  
##      Extent          Solidity        roundness       Compactness    
##  Min.   :0.5553   Min.   :0.9192   Min.   :0.4896   Min.   :0.6406  
##  1st Qu.:0.7186   1st Qu.:0.9857   1st Qu.:0.8321   1st Qu.:0.7625  
##  Median :0.7599   Median :0.9883   Median :0.8832   Median :0.8013  
##  Mean   :0.7497   Mean   :0.9871   Mean   :0.8733   Mean   :0.7999  
##  3rd Qu.:0.7869   3rd Qu.:0.9900   3rd Qu.:0.9169   3rd Qu.:0.8343  
##  Max.   :0.8662   Max.   :0.9947   Max.   :0.9907   Max.   :0.9873  
##   ShapeFactor1       ShapeFactor2        ShapeFactor3     ShapeFactor4   
##  Min.   :0.002778   Min.   :0.0005642   Min.   :0.4103   Min.   :0.9477  
##  1st Qu.:0.005900   1st Qu.:0.0011535   1st Qu.:0.5814   1st Qu.:0.9937  
##  Median :0.006645   Median :0.0016935   Median :0.6420   Median :0.9964  
##  Mean   :0.006564   Mean   :0.0017159   Mean   :0.6436   Mean   :0.9951  
##  3rd Qu.:0.007271   3rd Qu.:0.0021703   3rd Qu.:0.6960   3rd Qu.:0.9979  
##  Max.   :0.010451   Max.   :0.0036650   Max.   :0.9748   Max.   :0.9997  
##     Class          
##  Length:13611      
##  Class :character  
##  Mode  :character  
##                    
##                    
## 
  ggpairs(Dry_Bean_Dataset, aes(color = Class))
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

  ggpairs(Dry_Bean_Dataset, columns = c(1:8,17), aes(color = Class))
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

  ggpairs(Dry_Bean_Dataset, columns = c(9:17), aes(color = Class))
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

##Add Bayesian tests functions

#create function to conduct the Bayesian Sign Test
BayesianSignTest <- function(diffVector,rope_min,rope_max) {

  library(MCMCpack)

  samples <- 3000

  #build the vector 0.5 1 1 ....... 1 

  weights <- c(0.5,rep(1,length(diffVector)))

  #add the fake first observation in 0

  diffVector <- c (0, diffVector)  


  #for the moment we implement the sign test. Signedrank will follows

  probLeft <- mean (diffVector < rope_min)

  probRope <- mean (diffVector > rope_min & diffVector < rope_max)

  probRight <- mean (diffVector > rope_max)

  results = list ("probLeft"=probLeft, "probRope"=probRope,
                  
                  "probRight"=probRight)
  
  return (results)
}


##Create function to conduct Bayesian Signed Rank Test

BayesianSignedRank <- function(diffVector,rope_min,rope_max) {
  
  library(MCMCpack)
  
  samples <- 30000
  
  #build the vector 0.5 1 1 ....... 1
  weights <- c(0.5,rep(1,length(diffVector)))
  
  #add the fake first observation in 0
  diffVector <- c (0, diffVector)
  
  sampledWeights <- rdirichlet(samples,weights)
  
  winLeft <- vector(length = samples)
  winRope <- vector(length = samples)
  winRight <- vector(length = samples)
  
  for (rep in 1:samples){
    currentWeights <- sampledWeights[rep,]
    for (i in 1:length(currentWeights)){
      for (j in 1:length(currentWeights)){
        product= currentWeights[i] * currentWeights[j]
        if (diffVector[i]+diffVector[j] > (2*rope_max) ) {
          winRight[rep] <- winRight[rep] + product
        }
        else if (diffVector[i]+diffVector[j] > (2*rope_min) ) {
          winRope[rep] <- winRope[rep] + product
        }
        else {
          winLeft[rep] <- winLeft[rep] + product
        }

      }
    }
    maxWins=max(winRight[rep],winRope[rep],winLeft[rep])
    winners = (winRight[rep]==maxWins)*1 + (winRope[rep]==maxWins)*1 + (winLeft[rep]==maxWins)*1
    winRight[rep] <- (winRight[rep]==maxWins)*1/winners
    winRope[rep] <- (winRope[rep]==maxWins)*1/winners
    winLeft[rep] <- (winLeft[rep]==maxWins)*1/winners
  }
  
  
  results = list ("winLeft"=mean(winLeft), "winRope"=mean(winRope),
                  "winRight"=mean(winRight) )
  return (results)
  
}


#Create function to conduct the Bayesian Correlated t.test

#diff_a_b is a vector of differences between the two classifiers, on each fold of cross-validation.
#If you have done 10 runs of 10-folds cross-validation, you have 100 results for each classifier.
#You should have run cross-validation on the same folds for the two classifiers.
#Then diff_a_b is the difference fold-by-fold.

#rho is the correlation of the cross-validation results: 1/(number of folds)
#rope_min and rope_max are the lower and the upper bound of the rope
 
correlatedBayesianTtest <- function(diff_a_b,rho,rope_min,rope_max){
   if (rope_max < rope_min){
     stop("rope_max should be larger than rope_min")
   }
     
  delta <- mean(diff_a_b)
  n <- length(diff_a_b)
  df <- n-1
  stdX <- sd(diff_a_b)
  sp <- sd(diff_a_b)*sqrt(1/n + rho/(1-rho))
  p.left <- pt((rope_min - delta)/sp, df)
  p.rope <- pt((rope_max - delta)/sp, df)-p.left
  results <- list('left'=p.left,'rope'=p.rope,'right'=1-p.left-p.rope)
  return (results)
}
set.seed(16974)
###Prepare drybean dataset for One hot encoding if necessary and Persistent homology.
##One hot encoding for drybean dataset
library(caret)

#define one-hot encoding function
dummy_drybean <- dummyVars(" ~ .", data=Dry_Bean_Dataset)

#perform one-hot encoding on data frame
dry_bean_dataset_one_hot_df <- data.frame(predict(dummy_drybean, newdata=Dry_Bean_Dataset))
summary(dry_bean_dataset_one_hot_df)
##       Area          Perimeter      MajorAxisLength MinorAxisLength
##  Min.   : 20420   Min.   : 524.7   Min.   :183.6   Min.   :122.5  
##  1st Qu.: 36328   1st Qu.: 703.5   1st Qu.:253.3   1st Qu.:175.8  
##  Median : 44652   Median : 794.9   Median :296.9   Median :192.4  
##  Mean   : 53048   Mean   : 855.3   Mean   :320.1   Mean   :202.3  
##  3rd Qu.: 61332   3rd Qu.: 977.2   3rd Qu.:376.5   3rd Qu.:217.0  
##  Max.   :254616   Max.   :1985.4   Max.   :738.9   Max.   :460.2  
##   AspectRation    Eccentricity      ConvexArea     EquivDiameter  
##  Min.   :1.025   Min.   :0.2190   Min.   : 20684   Min.   :161.2  
##  1st Qu.:1.432   1st Qu.:0.7159   1st Qu.: 36714   1st Qu.:215.1  
##  Median :1.551   Median :0.7644   Median : 45178   Median :238.4  
##  Mean   :1.583   Mean   :0.7509   Mean   : 53768   Mean   :253.1  
##  3rd Qu.:1.707   3rd Qu.:0.8105   3rd Qu.: 62294   3rd Qu.:279.4  
##  Max.   :2.430   Max.   :0.9114   Max.   :263261   Max.   :569.4  
##      Extent          Solidity        roundness       Compactness    
##  Min.   :0.5553   Min.   :0.9192   Min.   :0.4896   Min.   :0.6406  
##  1st Qu.:0.7186   1st Qu.:0.9857   1st Qu.:0.8321   1st Qu.:0.7625  
##  Median :0.7599   Median :0.9883   Median :0.8832   Median :0.8013  
##  Mean   :0.7497   Mean   :0.9871   Mean   :0.8733   Mean   :0.7999  
##  3rd Qu.:0.7869   3rd Qu.:0.9900   3rd Qu.:0.9169   3rd Qu.:0.8343  
##  Max.   :0.8662   Max.   :0.9947   Max.   :0.9907   Max.   :0.9873  
##   ShapeFactor1       ShapeFactor2        ShapeFactor3     ShapeFactor4   
##  Min.   :0.002778   Min.   :0.0005642   Min.   :0.4103   Min.   :0.9477  
##  1st Qu.:0.005900   1st Qu.:0.0011535   1st Qu.:0.5814   1st Qu.:0.9937  
##  Median :0.006645   Median :0.0016935   Median :0.6420   Median :0.9964  
##  Mean   :0.006564   Mean   :0.0017159   Mean   :0.6436   Mean   :0.9951  
##  3rd Qu.:0.007271   3rd Qu.:0.0021703   3rd Qu.:0.6960   3rd Qu.:0.9979  
##  Max.   :0.010451   Max.   :0.0036650   Max.   :0.9748   Max.   :0.9997  
##  ClassBARBUNYA      ClassBOMBAY        ClassCALI      ClassDERMASON   
##  Min.   :0.00000   Min.   :0.00000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.00000   1st Qu.:0.00000   1st Qu.:0.0000   1st Qu.:0.0000  
##  Median :0.00000   Median :0.00000   Median :0.0000   Median :0.0000  
##  Mean   :0.09713   Mean   :0.03835   Mean   :0.1198   Mean   :0.2605  
##  3rd Qu.:0.00000   3rd Qu.:0.00000   3rd Qu.:0.0000   3rd Qu.:1.0000  
##  Max.   :1.00000   Max.   :1.00000   Max.   :1.0000   Max.   :1.0000  
##    ClassHOROZ       ClassSEKER       ClassSIRA     
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.0000   1st Qu.:0.0000   1st Qu.:0.0000  
##  Median :0.0000   Median :0.0000   Median :0.0000  
##  Mean   :0.1417   Mean   :0.1489   Mean   :0.1937  
##  3rd Qu.:0.0000   3rd Qu.:0.0000   3rd Qu.:0.0000  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000
dry_bean_dataset_one_hot_1000_df <- dry_bean_dataset_one_hot_df[sample(nrow(dry_bean_dataset_one_hot_df), size = 1000, replace = FALSE), ]
head(str(dry_bean_dataset_one_hot_1000_df))
## 'data.frame':    1000 obs. of  23 variables:
##  $ Area           : num  95754 43864 22144 27940 53196 ...
##  $ Perimeter      : num  1182 799 558 615 905 ...
##  $ MajorAxisLength: num  453 303 199 227 364 ...
##  $ MinorAxisLength: num  273 184 143 157 187 ...
##  $ AspectRation   : num  1.66 1.65 1.39 1.45 1.95 ...
##  $ Eccentricity   : num  0.799 0.794 0.695 0.723 0.859 ...
##  $ ConvexArea     : num  97441 44336 22445 28256 53781 ...
##  $ EquivDiameter  : num  349 236 168 189 260 ...
##  $ Extent         : num  0.749 0.733 0.72 0.808 0.775 ...
##  $ Solidity       : num  0.983 0.989 0.987 0.989 0.989 ...
##  $ roundness      : num  0.861 0.863 0.895 0.929 0.817 ...
##  $ Compactness    : num  0.771 0.779 0.843 0.83 0.715 ...
##  $ ShapeFactor1   : num  0.00473 0.00692 0.00899 0.00813 0.00685 ...
##  $ ShapeFactor2   : num  0.00103 0.00157 0.0028 0.00238 0.0011 ...
##  $ ShapeFactor3   : num  0.595 0.607 0.711 0.689 0.511 ...
##  $ ShapeFactor4   : num  0.988 0.998 0.989 0.998 0.996 ...
##  $ ClassBARBUNYA  : num  0 0 0 0 0 0 0 0 0 0 ...
##  $ ClassBOMBAY    : num  0 0 0 0 0 0 0 0 0 0 ...
##  $ ClassCALI      : num  1 0 0 0 0 1 0 0 0 1 ...
##  $ ClassDERMASON  : num  0 0 1 1 0 0 0 0 1 0 ...
##  $ ClassHOROZ     : num  0 0 0 0 1 0 0 0 0 0 ...
##  $ ClassSEKER     : num  0 0 0 0 0 0 0 1 0 0 ...
##  $ ClassSIRA      : num  0 1 0 0 0 0 1 0 0 0 ...
## NULL
##Persistent Homology of DryBean dataset

# calculate persistent homology for DryBean Dataset
phom_drybean_df <- calculate_homology(dry_bean_dataset_one_hot_1000_df)

# plot barcode for DryBean Dataset
plot_barcode(phom_drybean_df)

# plot persistent diagram of DryBean Dataset
plot_persist(phom_drybean_df)

#####———————————————MAPPER ALGORITHM————————————————

#Prepare Dry Bean dataset for Mapper 1D algorithm

##Two Filter Functions PCA & KDE

#Prepare linear PCA as a filter function by centering and scaling dataset first on all one hot df dataset
b<- prcomp(dry_bean_dataset_one_hot_df, center=TRUE, scale=TRUE)
ts_dry_bean_pca_b <- as.data.frame(predict(b, dry_bean_dataset_one_hot_df))

#Conduct kernel density estimator as a filter function on 4 of 6
filter.kde <- kde(dry_bean_dataset_one_hot_df[,1:4],H=diag(1,nrow = 4),eval.points = dry_bean_dataset_one_hot_df[,1:4])$estimate


###*** dry_bean_dataset PCA  Mapper 5 intervals, 60% overlap, 5 bins
##*** dry_bean_dataset PCA Mapper 5 intervals, 60% overlap, 5 bins

m_dry_bean_dataset_5.60.5 <- mapper1D(
     distance_matrix = dist(dry_bean_dataset_one_hot_df),
     filter_values = c(ts_dry_bean_pca_b$PC1),
     num_intervals = 5,
     percent_overlap = 60,
     num_bins_when_clustering = 5)


g_dry_bean_dataset_5.60.5 <- graph.adjacency(m_dry_bean_dataset_5.60.5$adjacency, mode="undirected")
## Warning: `graph.adjacency()` was deprecated in igraph 2.0.0.
## ℹ Please use `graph_from_adjacency_matrix()` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.
plot(g_dry_bean_dataset_5.60.5, layout = layout.auto(g_dry_bean_dataset_5.60.5))
## Warning: `layout.auto()` was deprecated in igraph 2.0.0.
## ℹ Please use `layout_nicely()` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.

head(str(m_dry_bean_dataset_5.60.5$level_of_vertex))
##  int [1:5] 1 2 3 4 5
## NULL
head(str(m_dry_bean_dataset_5.60.5$vertices_in_level))
## List of 5
##  $ : num 1
##  $ : num 2
##  $ : num 3
##  $ : num 4
##  $ : num 5
## NULL
head(str(m_dry_bean_dataset_5.60.5$points_in_vertex))
## List of 5
##  $ : int [1:8547] 1 2 3 4 5 6 7 8 9 10 ...
##  $ : int [1:10852] 25 43 108 198 211 272 279 294 369 374 ...
##  $ : int [1:5983] 272 1924 1973 1984 2010 2017 2018 2025 2028 2031 ...
##  $ : int [1:3037] 272 2216 2236 2283 2346 2383 2407 2421 2432 2437 ...
##  $ : int [1:505] 3357 3361 3364 3365 3367 3368 3369 3370 3372 3374 ...
## NULL
my_resolution = 100
my_palette = colorRampPalette(c('red','green','lightblue'))
my_max = max(m_dry_bean_dataset_5.60.5$level_of_vertex, na.rm=TRUE)
my_vector = m_dry_bean_dataset_5.60.5$level_of_vertex / my_max

my_colors = my_palette(my_resolution)[as.numeric(cut(
                       my_vector, breaks=my_resolution))]

g_dry_bean_dataset_5.60.5 <- graph.adjacency(m_dry_bean_dataset_5.60.5$adjacency, mode="undirected")
vertex_size <- unlist(lapply(m_dry_bean_dataset_5.60.5$points_in_vertex,
                             function(x) length(x)))

plot(g_dry_bean_dataset_5.60.5, layout = layout.auto(g_dry_bean_dataset_5.60.5),
     vertex.size = 30*log(vertex_size)/
     max(log(vertex_size)),
     vertex.color = my_colors)

m_dry_bean_dataset_5.60.5.n1<-m_dry_bean_dataset_5.60.5$points_in_vertex[1]
    m_dry_bean_dataset_5.60.5.n1.vec<-as.vector(unlist(m_dry_bean_dataset_5.60.5.n1))
m_dry_bean_dataset_5.60.5.n2<-m_dry_bean_dataset_5.60.5$points_in_vertex[2]
    m_dry_bean_dataset_5.60.5.n2.vec<-as.vector(unlist(m_dry_bean_dataset_5.60.5.n2)) 
m_dry_bean_dataset_5.60.5.n3<-m_dry_bean_dataset_5.60.5$points_in_vertex[3]
    m_dry_bean_dataset_5.60.5.n3.vec<-as.vector(unlist(m_dry_bean_dataset_5.60.5.n3))
m_dry_bean_dataset_5.60.5.n4<-m_dry_bean_dataset_5.60.5$points_in_vertex[4]
    m_dry_bean_dataset_5.60.5.n4.vec<-as.vector(unlist(m_dry_bean_dataset_5.60.5.n4)) 
m_dry_bean_dataset_5.60.5.n5<-m_dry_bean_dataset_5.60.5$points_in_vertex[5]
    m_dry_bean_dataset_5.60.5.n5.vec<-as.vector(unlist(m_dry_bean_dataset_5.60.5.n5))

##map the ID’s of each Mapper vertex point to the actual dry_bean_dataset One Hot DF1 dataset
tda.m_dry_bean_dataset_5.60.5.n1.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.60.5.n1.vec,]
tda.m_dry_bean_dataset_5.60.5.n2.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.60.5.n2.vec,]
tda.m_dry_bean_dataset_5.60.5.n3.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.60.5.n3.vec,]
tda.m_dry_bean_dataset_5.60.5.n4.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.60.5.n4.vec,]
tda.m_dry_bean_dataset_5.60.5.n5.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.60.5.n5.vec,]


###*** dry_bean_dataset Mapper 5 intervals, 50% overlap, 5 bins

m_dry_bean_dataset_5.50.5 <- mapper1D(
     distance_matrix = dist(dry_bean_dataset_one_hot_df),
     filter_values = c(ts_dry_bean_pca_b$PC1),
     num_intervals = 5,
     percent_overlap = 50,
     num_bins_when_clustering = 5)


g_dry_bean_dataset_5.50.5 <- graph.adjacency(m_dry_bean_dataset_5.50.5$adjacency, mode="undirected")
plot(g_dry_bean_dataset_5.50.5, layout = layout.auto(g_dry_bean_dataset_5.50.5))

head(str(m_dry_bean_dataset_5.50.5$level_of_vertex))
##  int [1:5] 1 2 3 4 5
## NULL
head(str(m_dry_bean_dataset_5.50.5$vertices_in_level))
## List of 5
##  $ : num 1
##  $ : num 2
##  $ : num 3
##  $ : num 4
##  $ : num 5
## NULL
head(str(m_dry_bean_dataset_5.50.5$points_in_vertex))
## List of 5
##  $ : int [1:7839] 1 2 3 4 5 6 7 8 9 10 ...
##  $ : int [1:9515] 272 279 294 402 413 431 433 446 457 548 ...
##  $ : int [1:5355] 272 2010 2028 2031 2035 2037 2040 2043 2046 2052 ...
##  $ : int [1:1590] 2236 2455 2463 2512 2515 2555 2647 2663 2664 2721 ...
##  $ : int [1:417] 3369 3375 3398 3399 3406 3410 3411 3412 3413 3414 ...
## NULL
my_resolution = 100
my_palette = colorRampPalette(c('red','green','lightblue'))
my_max = max(m_dry_bean_dataset_5.50.5$level_of_vertex, na.rm=TRUE)
my_vector = m_dry_bean_dataset_5.50.5$level_of_vertex / my_max

my_colors = my_palette(my_resolution)[as.numeric(cut(
                       my_vector, breaks=my_resolution))]

g_dry_bean_dataset_5.50.5 <- graph.adjacency(m_dry_bean_dataset_5.50.5$adjacency, mode="undirected")
vertex_size <- unlist(lapply(m_dry_bean_dataset_5.50.5$points_in_vertex,
                             function(x) length(x)))

plot(g_dry_bean_dataset_5.50.5, layout = layout.auto(g_dry_bean_dataset_5.50.5),
     vertex.size = 30*log(vertex_size)/
     max(log(vertex_size)),
     vertex.color = my_colors)

##Extract the ID observations of each mapper output vertex
m_dry_bean_dataset_5.50.5.n1<-m_dry_bean_dataset_5.50.5$points_in_vertex[1]
    m_dry_bean_dataset_5.50.5.n1.vec<-as.vector(unlist(m_dry_bean_dataset_5.50.5.n1))
m_dry_bean_dataset_5.50.5.n2<-m_dry_bean_dataset_5.50.5$points_in_vertex[2]
    m_dry_bean_dataset_5.50.5.n2.vec<-as.vector(unlist(m_dry_bean_dataset_5.50.5.n2)) 
m_dry_bean_dataset_5.50.5.n3<-m_dry_bean_dataset_5.50.5$points_in_vertex[3]
    m_dry_bean_dataset_5.50.5.n3.vec<-as.vector(unlist(m_dry_bean_dataset_5.50.5.n3))
m_dry_bean_dataset_5.50.5.n4<-m_dry_bean_dataset_5.50.5$points_in_vertex[4]
    m_dry_bean_dataset_5.50.5.n4.vec<-as.vector(unlist(m_dry_bean_dataset_5.50.5.n4)) 
m_dry_bean_dataset_5.50.5.n5<-m_dry_bean_dataset_5.50.5$points_in_vertex[5]
    m_dry_bean_dataset_5.50.5.n5.vec<-as.vector(unlist(m_dry_bean_dataset_5.50.5.n5))

##map the ID’s of each Mapper vertex point to the actual dry_bean_dataset One Hot DF1 dataset
tda.m_dry_bean_dataset_5.50.5.n1.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.50.5.n1.vec,]
tda.m_dry_bean_dataset_5.50.5.n2.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.50.5.n2.vec,]
tda.m_dry_bean_dataset_5.50.5.n3.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.50.5.n3.vec,]
tda.m_dry_bean_dataset_5.50.5.n4.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.50.5.n4.vec,]
tda.m_dry_bean_dataset_5.50.5.n5.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.50.5.n5.vec,]

##*** dry_bean_dataset Mapper 5 intervals, 40% overlap, 5 bins

m_dry_bean_dataset_5.40.5 <- mapper1D(
     distance_matrix = dist(dry_bean_dataset_one_hot_df),
     filter_values = c(ts_dry_bean_pca_b$PC1),
     num_intervals = 5,
     percent_overlap = 40,
     num_bins_when_clustering = 5)


g_dry_bean_dataset_5.40.5 <- graph.adjacency(m_dry_bean_dataset_5.40.5$adjacency, mode="undirected")
plot(g_dry_bean_dataset_5.40.5, layout = layout.auto(g_dry_bean_dataset_5.40.5))

head(str(m_dry_bean_dataset_5.40.5$level_of_vertex))
##  int [1:5] 1 2 3 4 5
## NULL
head(str(m_dry_bean_dataset_5.40.5$vertices_in_level))
## List of 5
##  $ : num 1
##  $ : num 2
##  $ : num 3
##  $ : num 4
##  $ : num 5
## NULL
head(str(m_dry_bean_dataset_5.40.5$points_in_vertex))
## List of 5
##  $ : int [1:6835] 1 2 3 4 5 6 7 8 9 10 ...
##  $ : int [1:8024] 272 279 431 433 457 646 667 713 759 798 ...
##  $ : int [1:5008] 272 2028 2046 2054 2055 2056 2059 2060 2063 2064 ...
##  $ : int [1:894] 2647 2935 2951 2987 3064 3066 3081 3082 3084 3093 ...
##  $ : int [1:342] 3375 3424 3427 3428 3430 3435 3437 3450 3453 3456 ...
## NULL
my_resolution = 100
my_palette = colorRampPalette(c('red','green','lightblue'))
my_max = max(m_dry_bean_dataset_5.40.5$level_of_vertex, na.rm=TRUE)
my_vector = m_dry_bean_dataset_5.40.5$level_of_vertex / my_max

my_colors = my_palette(my_resolution)[as.numeric(cut(
                       my_vector, breaks=my_resolution))]

g_dry_bean_dataset_5.40.5 <- graph.adjacency(m_dry_bean_dataset_5.40.5$adjacency, mode="undirected")
vertex_size <- unlist(lapply(m_dry_bean_dataset_5.40.5$points_in_vertex,
                             function(x) length(x)))

plot(g_dry_bean_dataset_5.50.5, layout = layout.auto(g_dry_bean_dataset_5.40.5),
     vertex.size = 30*log(vertex_size)/
     max(log(vertex_size)),
     vertex.color = my_colors)

m_dry_bean_dataset_5.40.5.n1<-m_dry_bean_dataset_5.40.5$points_in_vertex[1]
    m_dry_bean_dataset_5.40.5.n1.vec<-as.vector(unlist(m_dry_bean_dataset_5.40.5.n1))
m_dry_bean_dataset_5.40.5.n2<-m_dry_bean_dataset_5.40.5$points_in_vertex[2]
    m_dry_bean_dataset_5.40.5.n2.vec<-as.vector(unlist(m_dry_bean_dataset_5.40.5.n2)) 
m_dry_bean_dataset_5.40.5.n3<-m_dry_bean_dataset_5.40.5$points_in_vertex[3]
    m_dry_bean_dataset_5.40.5.n3.vec<-as.vector(unlist(m_dry_bean_dataset_5.40.5.n3))
m_dry_bean_dataset_5.40.5.n4<-m_dry_bean_dataset_5.40.5$points_in_vertex[4]
    m_dry_bean_dataset_5.40.5.n4.vec<-as.vector(unlist(m_dry_bean_dataset_5.40.5.n4)) 
m_dry_bean_dataset_5.40.5.n5<-m_dry_bean_dataset_5.40.5$points_in_vertex[5]
    m_dry_bean_dataset_5.40.5.n5.vec<-as.vector(unlist(m_dry_bean_dataset_5.40.5.n5))

##map the ID’s of each Mapper vertex point to the actual dry_bean_dataset One Hot DF1 dataset
tda.m_dry_bean_dataset_5.40.5.n1.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.40.5.n1.vec,]
tda.m_dry_bean_dataset_5.40.5.n2.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.40.5.n2.vec,]
tda.m_dry_bean_dataset_5.40.5.n3.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.40.5.n3.vec,]
tda.m_dry_bean_dataset_5.40.5.n4.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.40.5.n4.vec,]
tda.m_dry_bean_dataset_5.40.5.n5.vec<-Dry_Bean_Dataset[m_dry_bean_dataset_5.40.5.n5.vec,]


##*** dry_bean_dataset Mapper KDE Filter 5 intervals, 60% overlap, 5 bins

m_kde_dry_bean_dataset_5.60.5 <- mapper1D(
      distance_matrix = dist(dry_bean_dataset_one_hot_df),
      filter_values = c(filter.kde),
      num_intervals = 5,
      percent_overlap = 60,
      num_bins_when_clustering = 5)


g_kde_dry_bean_dataset_5.60.5 <- graph.adjacency(m_kde_dry_bean_dataset_5.60.5$adjacency, mode="undirected")
plot(g_kde_dry_bean_dataset_5.60.5, layout = layout.auto(g_kde_dry_bean_dataset_5.60.5))

head(str(m_kde_dry_bean_dataset_5.60.5$level_of_vertex))
##  int [1:5] 1 2 3 4 5
## NULL
head(str(m_kde_dry_bean_dataset_5.60.5$vertices_in_level))
## List of 5
##  $ : num 1
##  $ : num 2
##  $ : num 3
##  $ : num 4
##  $ : num 5
## NULL
head(str(m_kde_dry_bean_dataset_5.60.5$points_in_vertex))
## List of 5
##  $ : int [1:9688] 1 2 3 4 5 6 7 8 9 10 ...
##  $ : int [1:8239] 1 3 4 6 7 8 9 10 11 12 ...
##  $ : int [1:4917] 25 39 43 68 96 102 104 108 114 142 ...
##  $ : int [1:2448] 198 211 279 294 307 347 355 360 369 371 ...
##  $ : int [1:1349] 402 431 433 438 443 456 457 488 531 536 ...
## NULL
my_resolution = 100
my_palette = colorRampPalette(c('red','green','lightblue'))
my_max = max(m_kde_dry_bean_dataset_5.60.5$level_of_vertex, na.rm=TRUE)
my_vector = m_kde_dry_bean_dataset_5.60.5$level_of_vertex / my_max

my_colors = my_palette(my_resolution)[as.numeric(cut(
                       my_vector, breaks=my_resolution))]

g_kde_dry_bean_dataset_5.50.5 <- graph.adjacency(m_kde_dry_bean_dataset_5.60.5$adjacency, mode="undirected")
vertex_size <- unlist(lapply(m_kde_dry_bean_dataset_5.60.5$points_in_vertex,
                             function(x) length(x)))

plot(g_kde_dry_bean_dataset_5.60.5, layout = layout.auto(g_kde_dry_bean_dataset_5.60.5),
     vertex.size = 30*log(vertex_size)/
     max(log(vertex_size)),
     vertex.color = my_colors)

##Extract the ID observations of each mapper output vertex
m_kde_dry_bean_dataset_5.60.5.n1<-m_kde_dry_bean_dataset_5.60.5$points_in_vertex[1]
    m_kde_dry_bean_dataset_5.60.5.n1.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.60.5.n1))
m_kde_dry_bean_dataset_5.60.5.n2<-m_kde_dry_bean_dataset_5.60.5$points_in_vertex[2]
    m_kde_dry_bean_dataset_5.60.5.n2.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.60.5.n2)) 
m_kde_dry_bean_dataset_5.60.5.n3<-m_kde_dry_bean_dataset_5.60.5$points_in_vertex[3]
    m_kde_dry_bean_dataset_5.60.5.n3.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.60.5.n3))
m_kde_dry_bean_dataset_5.60.5.n4<-m_kde_dry_bean_dataset_5.60.5$points_in_vertex[4]
    m_kde_dry_bean_dataset_5.60.5.n4.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.60.5.n4)) 
m_kde_dry_bean_dataset_5.60.5.n5<-m_kde_dry_bean_dataset_5.60.5 $points_in_vertex[5]
    m_kde_dry_bean_dataset_5.60.5.n5.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.60.5.n5))

##map the ID’s of each Mapper vertex point to the actual dry_bean_dataset One Hot DF1 dataset
tda.m_kde_dry_bean_dataset_5.60.5.n1.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.60.5.n1.vec,]
tda.m_kde_dry_bean_dataset_5.60.5.n2.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.60.5.n2.vec,]
tda.m_kde_dry_bean_dataset_5.60.5.n3.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.60.5.n3.vec,]
tda.m_kde_dry_bean_dataset_5.60.5.n4.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.60.5.n4.vec,]
tda.m_kde_dry_bean_dataset_5.60.5.n5.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.60.5.n5.vec,]



##*** dry_bean_dataset Mapper KDE Filter 5 intervals, 50% overlap, 5 bins

m_kde_dry_bean_dataset_5.50.5 <- mapper1D(
     distance_matrix = dist(dry_bean_dataset_one_hot_df),
     filter_values = c(filter.kde),
     num_intervals = 5,
     percent_overlap = 50,
     num_bins_when_clustering = 5)

g_kde_dry_bean_dataset_5.50.5 <- graph.adjacency(m_kde_dry_bean_dataset_5.50.5$adjacency, mode="undirected")
plot(g_kde_dry_bean_dataset_5.50.5, layout = layout.auto(g_kde_dry_bean_dataset_5.50.5))

head(str(m_kde_dry_bean_dataset_5.50.5$level_of_vertex))
##  int [1:5] 1 2 3 4 5
## NULL
head(str(m_kde_dry_bean_dataset_5.50.5$vertices_in_level))
## List of 5
##  $ : num 1
##  $ : num 2
##  $ : num 3
##  $ : num 4
##  $ : num 5
## NULL
head(str(m_kde_dry_bean_dataset_5.50.5$points_in_vertex))
## List of 5
##  $ : int [1:8473] 1 2 3 4 5 6 7 8 9 10 ...
##  $ : int [1:7582] 1 3 4 6 7 8 9 10 11 13 ...
##  $ : int [1:4149] 25 43 68 96 102 108 151 158 159 162 ...
##  $ : int [1:2024] 279 294 369 374 376 385 388 401 402 409 ...
##  $ : int [1:989] 431 457 488 536 548 582 587 590 593 605 ...
## NULL
my_resolution = 100
my_palette = colorRampPalette(c('red','green','lightblue'))
my_max = max(m_kde_dry_bean_dataset_5.50.5$level_of_vertex, na.rm=TRUE)
my_vector = m_kde_dry_bean_dataset_5.50.5$level_of_vertex / my_max

my_colors = my_palette(my_resolution)[as.numeric(cut(
                       my_vector, breaks=my_resolution))]

g_kde_dry_bean_dataset_5.50.5 <- graph.adjacency(m_kde_dry_bean_dataset_5.50.5$adjacency, mode="undirected")
vertex_size <- unlist(lapply(m_kde_dry_bean_dataset_5.50.5$points_in_vertex,
                             function(x) length(x)))

plot(g_kde_dry_bean_dataset_5.50.5, layout = layout.auto(g_kde_dry_bean_dataset_5.50.5),
     vertex.size = 30*log(vertex_size)/
     max(log(vertex_size)),
     vertex.color = my_colors)

##Extract the ID observations of each mapper output vertex
m_kde_dry_bean_dataset_5.50.5.n1<-m_kde_dry_bean_dataset_5.50.5$points_in_vertex[1]
    m_kde_dry_bean_dataset_5.50.5.n1.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.50.5.n1))
m_kde_dry_bean_dataset_5.50.5.n2<-m_kde_dry_bean_dataset_5.50.5$points_in_vertex[2]
    m_kde_dry_bean_dataset_5.50.5.n2.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.50.5.n2)) 
m_kde_dry_bean_dataset_5.50.5.n3<-m_kde_dry_bean_dataset_5.50.5$points_in_vertex[3]
    m_kde_dry_bean_dataset_5.50.5.n3.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.50.5.n3))
m_kde_dry_bean_dataset_5.50.5.n4<-m_kde_dry_bean_dataset_5.50.5$points_in_vertex[4]
    m_kde_dry_bean_dataset_5.50.5.n4.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.50.5.n4)) 
m_kde_dry_bean_dataset_5.50.5.n5<-m_kde_dry_bean_dataset_5.50.5 $points_in_vertex[5]
    m_kde_dry_bean_dataset_5.50.5.n5.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.50.5.n5))

##map the ID’s of each Mapper vertex point to the actual dry_bean_dataset One Hot DF1 dataset
tda.m_kde_dry_bean_dataset_5.50.5.n1.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.50.5.n1.vec,]
tda.m_kde_dry_bean_dataset_5.50.5.n2.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.50.5.n2.vec,]
tda.m_kde_dry_bean_dataset_5.50.5.n3.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.50.5.n3.vec,]
tda.m_kde_dry_bean_dataset_5.50.5.n4.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.50.5.n4.vec,]
tda.m_kde_dry_bean_dataset_5.50.5.n5.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.50.5.n5.vec,]




##*** dry_bean_dataset Mapper KDE 5 intervals, 40% overlap, 5 bins

m_kde_dry_bean_dataset_5.40.5 <- mapper1D(
     distance_matrix = dist(dry_bean_dataset_one_hot_df),
     filter_values = c(filter.kde),
     num_intervals = 5,
     percent_overlap = 40,
     num_bins_when_clustering = 5)


g_kde_dry_bean_dataset_5.40.5 <- graph.adjacency(m_kde_dry_bean_dataset_5.40.5$adjacency, mode="undirected")
plot(g_kde_dry_bean_dataset_5.40.5, layout = layout.auto(g_kde_dry_bean_dataset_5.40.5))

head(str(m_kde_dry_bean_dataset_5.40.5$level_of_vertex))
##  int [1:5] 1 2 3 4 5
## NULL
head(str(m_kde_dry_bean_dataset_5.40.5$vertices_in_level))
## List of 5
##  $ : num 1
##  $ : num 2
##  $ : num 3
##  $ : num 4
##  $ : num 5
## NULL
head(str(m_kde_dry_bean_dataset_5.40.5$points_in_vertex))
## List of 5
##  $ : int [1:7503] 1 2 3 4 5 6 7 8 9 10 ...
##  $ : int [1:7002] 1 3 4 6 8 9 10 11 13 14 ...
##  $ : int [1:3511] 25 108 159 183 197 198 202 206 209 211 ...
##  $ : int [1:1759] 294 369 374 376 401 402 409 413 431 433 ...
##  $ : int [1:774] 548 593 615 616 618 631 633 638 640 646 ...
## NULL
my_resolution = 100
my_palette = colorRampPalette(c('red','green','lightblue'))
my_max = max(m_kde_dry_bean_dataset_5.40.5$level_of_vertex, na.rm=TRUE)
my_vector = m_kde_dry_bean_dataset_5.40.5$level_of_vertex / my_max

my_colors = my_palette(my_resolution)[as.numeric(cut(
                       my_vector, breaks=my_resolution))]

g_kde_dry_bean_dataset_5.40.5 <- graph.adjacency(m_kde_dry_bean_dataset_5.40.5$adjacency, mode="undirected")
vertex_size <- unlist(lapply(m_kde_dry_bean_dataset_5.40.5$points_in_vertex,
                             function(x) length(x)))

plot(g_kde_dry_bean_dataset_5.40.5, layout = layout.auto(g_kde_dry_bean_dataset_5.40.5),
     vertex.size = 30*log(vertex_size)/
     max(log(vertex_size)),
     vertex.color = my_colors)

##Extract the ID observations of each mapper output vertex
m_kde_dry_bean_dataset_5.40.5.n1<-m_kde_dry_bean_dataset_5.40.5$points_in_vertex[1]
    m_kde_dry_bean_dataset_5.40.5.n1.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.40.5.n1))
m_kde_dry_bean_dataset_5.40.5.n2<-m_kde_dry_bean_dataset_5.40.5$points_in_vertex[2]
    m_kde_dry_bean_dataset_5.40.5.n2.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.40.5.n2)) 
m_kde_dry_bean_dataset_5.40.5.n3<-m_kde_dry_bean_dataset_5.40.5$points_in_vertex[3]
    m_kde_dry_bean_dataset_5.40.5.n3.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.40.5.n3))
m_kde_dry_bean_dataset_5.40.5.n4<-m_kde_dry_bean_dataset_5.40.5$points_in_vertex[4]
    m_kde_dry_bean_dataset_5.40.5.n4.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.40.5.n4)) 
m_kde_dry_bean_dataset_5.40.5.n5<-m_kde_dry_bean_dataset_5.40.5 $points_in_vertex[5]
    m_kde_dry_bean_dataset_5.40.5.n5.vec<-as.vector(unlist(m_kde_dry_bean_dataset_5.40.5.n5))

##map the ID’s of each Mapper vertex point to the actual dry_bean_dataset One Hot DF4 dataset
tda.m_kde_dry_bean_dataset_5.40.5.n1.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.40.5.n1.vec,]
tda.m_kde_dry_bean_dataset_5.40.5.n2.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.40.5.n2.vec,]
tda.m_kde_dry_bean_dataset_5.40.5.n3.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.40.5.n3.vec,]
tda.m_kde_dry_bean_dataset_5.40.5.n4.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.40.5.n4.vec,]
tda.m_kde_dry_bean_dataset_5.40.5.n5.vec<-Dry_Bean_Dataset[m_kde_dry_bean_dataset_5.40.5.n5.vec,]
library(caret)

trainIndex <- createDataPartition(Dry_Bean_Dataset$Class, p = .7, 
                                  list = FALSE, 
                                  times = 1)

head(trainIndex)
##      Resample1
## [1,]         2
## [2,]         4
## [3,]         5
## [4,]         6
## [5,]         7
## [6,]         8
Dry_Bean_DatasetTrain <- Dry_Bean_Dataset[ trainIndex,]
Dry_Bean_DatasetTest  <- Dry_Bean_Dataset[-trainIndex,]
#Train Control: k-Fold Cross-validation basis for all models 
fitControl <- trainControl(## 10-fold CV
                           method = "cv",
                           number = 3)
#Non-TDA-Assited
rfGrid<-expand.grid(mtry = (1:20)*50)
#Random Forest 
dryBeanRfFit <- train(as.factor(Class) ~ ., data = Dry_Bean_DatasetTrain, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'),                    
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
dryBeanRfFit
## Random Forest 
## 
## 9531 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6354, 6355, 6353 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.9265562  0.9111474
##    100  0.9256121  0.9100126
##    150  0.9251923  0.9094962
##    200  0.9259265  0.9103820
##    250  0.9262413  0.9107681
##    300  0.9252971  0.9096266
##    350  0.9251921  0.9095050
##    400  0.9256119  0.9100022
##    450  0.9247724  0.9089949
##    500  0.9261365  0.9106455
##    550  0.9260314  0.9105091
##    600  0.9246677  0.9088703
##    650  0.9252970  0.9096301
##    700  0.9250874  0.9093706
##    750  0.9250873  0.9093743
##    800  0.9258219  0.9102635
##    850  0.9252970  0.9096184
##    900  0.9257170  0.9101351
##    950  0.9256117  0.9100080
##   1000  0.9243527  0.9084797
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 50.
dryBeanRfFit$resample
##    Accuracy     Kappa Resample
## 1 0.9250866 0.9093960    Fold1
## 2 0.9238515 0.9078857    Fold3
## 3 0.9307305 0.9161604    Fold2
db_rf_fit_re<-dryBeanRfFit$resample[1]


summary(dryBeanRfFit)
##                 Length Class      Mode     
## call                5  -none-     call     
## type                1  -none-     character
## predicted        9531  factor     numeric  
## err.rate         4000  -none-     numeric  
## confusion          56  -none-     numeric  
## votes           66717  matrix     numeric  
## oob.times        9531  -none-     numeric  
## classes             7  -none-     character
## importance         16  -none-     numeric  
## importanceSD        0  -none-     NULL     
## localImportance     0  -none-     NULL     
## proximity           0  -none-     NULL     
## ntree               1  -none-     numeric  
## mtry                1  -none-     numeric  
## forest             14  -none-     list     
## y                9531  factor     numeric  
## test                0  -none-     NULL     
## inbag               0  -none-     NULL     
## xNames             16  -none-     character
## problemType         1  -none-     character
## tuneValue           1  data.frame list     
## obsLevels           7  -none-     character
## param               1  -none-     list
vip(dryBeanRfFit,25) + ggtitle("non-TDA-Assisted: RF")

# Predict outcome using model from training data based on testing data
predictions <- predict(dryBeanRfFit, newdata = Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_rf_cf<-confusionMatrix(data=predictions, as.factor(Dry_Bean_DatasetTest$Class))
db_rf_cf
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      363      0   15        0     3     4    7
##   BOMBAY          1    156    0        0     0     0    0
##   CALI           19      0  461        0    12     0    1
##   DERMASON        1      0    0      969     6    17   90
##   HOROZ           5      0    7        2   548     0   12
##   SEKER           1      0    1       21     0   573    5
##   SIRA            6      0    5       71     9    14  675
## 
## Overall Statistics
##                                          
##                Accuracy : 0.9179         
##                  95% CI : (0.909, 0.9261)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.9007         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91667       1.00000      0.9427          0.9116
## Specificity                  0.99213       0.99975      0.9911          0.9622
## Pos Pred Value               0.92602       0.99363      0.9351          0.8947
## Neg Pred Value               0.99105       1.00000      0.9922          0.9686
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08897       0.03824      0.1130          0.2375
## Detection Prevalence         0.09608       0.03848      0.1208          0.2654
## Balanced Accuracy            0.95440       0.99987      0.9669          0.9369
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9481       0.9424      0.8544
## Specificity                0.9926       0.9919      0.9681
## Pos Pred Value             0.9547       0.9534      0.8654
## Neg Pred Value             0.9914       0.9899      0.9652
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1343       0.1404      0.1654
## Detection Prevalence       0.1407       0.1473      0.1912
## Balanced Accuracy          0.9703       0.9672      0.9113
db_rf_cf$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.9178922      0.9006758      0.9090433      0.9261370      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_rf_cf_ov_acc<-db_rf_cf$overall[1]
db_rf_cf$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9166667   0.9921281      0.9260204      0.9910521 0.9260204
## Class: BOMBAY     1.0000000   0.9997452      0.9936306      1.0000000 0.9936306
## Class: CALI       0.9427403   0.9910888      0.9350913      0.9921940 0.9350913
## Class: DERMASON   0.9115710   0.9622141      0.8947368      0.9686353 0.8947368
## Class: HOROZ      0.9480969   0.9925757      0.9547038      0.9914432 0.9547038
## Class: SEKER      0.9424342   0.9919355      0.9534110      0.9899396 0.9534110
## Class: SIRA       0.8544304   0.9680851      0.8653846      0.9651515 0.8653846
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9166667 0.9213198 0.09705882     0.08897059
## Class: BOMBAY   1.0000000 0.9968051 0.03823529     0.03823529
## Class: CALI     0.9427403 0.9389002 0.11985294     0.11299020
## Class: DERMASON 0.9115710 0.9030755 0.26053922     0.23750000
## Class: HOROZ    0.9480969 0.9513889 0.14166667     0.13431373
## Class: SEKER    0.9424342 0.9478908 0.14901961     0.14044118
## Class: SIRA     0.8544304 0.8598726 0.19362745     0.16544118
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09607843         0.9543974
## Class: BOMBAY             0.03848039         0.9998726
## Class: CALI               0.12083333         0.9669146
## Class: DERMASON           0.26544118         0.9368926
## Class: HOROZ              0.14068627         0.9703363
## Class: SEKER              0.14730392         0.9671848
## Class: SIRA               0.19117647         0.9112577
db_rf_cf_pre_rec_f1<-db_rf_cf$byClass[5:7]


##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node1

DryBean_TDA_PC_5.50.5_n1_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n1.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'), 
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_PC_5.50.5_n1_RfFit0
## Random Forest 
## 
## 7839 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5224, 5226, 5228 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.9109577  0.8623916
##    100  0.9103201  0.8614161
##    150  0.9103200  0.8614826
##    200  0.9104481  0.8615998
##    250  0.9104481  0.8616920
##    300  0.9118504  0.8638404
##    350  0.9107030  0.8620755
##    400  0.9096821  0.8605097
##    450  0.9105749  0.8618106
##    500  0.9101924  0.8612533
##    550  0.9109578  0.8624083
##    600  0.9115956  0.8634139
##    650  0.9107035  0.8620544
##    700  0.9105756  0.8618413
##    750  0.9115960  0.8634190
##    800  0.9103197  0.8613922
##    850  0.9101927  0.8612581
##    900  0.9105752  0.8618085
##    950  0.9096826  0.8604303
##   1000  0.9107029  0.8620331
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 300.
DryBean_TDA_PC_5.50.5_n1_RfFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9116635 0.8636336    Fold1
## 2 0.9092302 0.8595162    Fold3
## 3 0.9146575 0.8683713    Fold2
db_tda_pc_5.50.5_n1_rf_fit0_re<-DryBean_TDA_PC_5.50.5_n1_RfFit0$resample[1]


summary(DryBean_TDA_PC_5.50.5_n1_RfFit0)
##                 Length Class      Mode     
## call                5  -none-     call     
## type                1  -none-     character
## predicted        7839  factor     numeric  
## err.rate         3500  -none-     numeric  
## confusion          42  -none-     numeric  
## votes           47034  matrix     numeric  
## oob.times        7839  -none-     numeric  
## classes             6  -none-     character
## importance         16  -none-     numeric  
## importanceSD        0  -none-     NULL     
## localImportance     0  -none-     NULL     
## proximity           0  -none-     NULL     
## ntree               1  -none-     numeric  
## mtry                1  -none-     numeric  
## forest             14  -none-     list     
## y                7839  factor     numeric  
## test                0  -none-     NULL     
## inbag               0  -none-     NULL     
## xNames             16  -none-     character
## problemType         1  -none-     character
## tuneValue           1  data.frame list     
## obsLevels           6  -none-     character
## param               1  -none-     list
vip(DryBean_TDA_PC_5.50.5_n1_RfFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n1_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_PC_5.50.5_n1_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n1_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n1_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n1_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      253     22  121        0    70     0    1
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    1        0     0     0    0
##   DERMASON        0      0    0     1063   198     0    1
##   HOROZ           0      0    0        0     8     0    0
##   SEKER         138    134  362        0    11   608   18
##   SIRA            5      0    5        0   291     0  770
## 
## Overall Statistics
##                                          
##                Accuracy : 0.6625         
##                  95% CI : (0.6478, 0.677)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.5837         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.63889       0.00000   0.0020450          1.0000
## Specificity                  0.94191       1.00000   1.0000000          0.9340
## Pos Pred Value               0.54176           NaN   1.0000000          0.8423
## Neg Pred Value               0.96042       0.96176   0.8803628          1.0000
## Prevalence                   0.09706       0.03824   0.1198529          0.2605
## Detection Rate               0.06201       0.00000   0.0002451          0.2605
## Detection Prevalence         0.11446       0.00000   0.0002451          0.3093
## Balanced Accuracy            0.79040       0.50000   0.5010225          0.9670
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity              0.013841       1.0000      0.9747
## Specificity              1.000000       0.8090      0.9085
## Pos Pred Value           1.000000       0.4784      0.7190
## Neg Pred Value           0.860020       1.0000      0.9934
## Prevalence               0.141667       0.1490      0.1936
## Detection Rate           0.001961       0.1490      0.1887
## Detection Prevalence     0.001961       0.3115      0.2625
## Balanced Accuracy        0.506920       0.9045      0.9416
db_tda_pc_5.50.5_n1_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      253     22  121        0    70     0    1
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    1        0     0     0    0
##   DERMASON        0      0    0     1063   198     0    1
##   HOROZ           0      0    0        0     8     0    0
##   SEKER         138    134  362        0    11   608   18
##   SIRA            5      0    5        0   291     0  770
## 
## Overall Statistics
##                                          
##                Accuracy : 0.6625         
##                  95% CI : (0.6478, 0.677)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.5837         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.63889       0.00000   0.0020450          1.0000
## Specificity                  0.94191       1.00000   1.0000000          0.9340
## Pos Pred Value               0.54176           NaN   1.0000000          0.8423
## Neg Pred Value               0.96042       0.96176   0.8803628          1.0000
## Prevalence                   0.09706       0.03824   0.1198529          0.2605
## Detection Rate               0.06201       0.00000   0.0002451          0.2605
## Detection Prevalence         0.11446       0.00000   0.0002451          0.3093
## Balanced Accuracy            0.79040       0.50000   0.5010225          0.9670
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity              0.013841       1.0000      0.9747
## Specificity              1.000000       0.8090      0.9085
## Pos Pred Value           1.000000       0.4784      0.7190
## Neg Pred Value           0.860020       1.0000      0.9934
## Prevalence               0.141667       0.1490      0.1936
## Detection Rate           0.001961       0.1490      0.1887
## Detection Prevalence     0.001961       0.3115      0.2625
## Balanced Accuracy        0.506920       0.9045      0.9416
db_tda_pc_5.50.5_n1_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.6625000      0.5837164      0.6477557      0.6770113      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n1_rf_cf0_ov_acc<-db_tda_pc_5.50.5_n1_rf_cf0$overall[1]
db_tda_pc_5.50.5_n1_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA  0.63888889   0.9419110      0.5417559      0.9604207 0.5417559
## Class: BOMBAY    0.00000000   1.0000000            NaN      0.9617647        NA
## Class: CALI      0.00204499   1.0000000      1.0000000      0.8803628 1.0000000
## Class: DERMASON  1.00000000   0.9340404      0.8423138      1.0000000 0.8423138
## Class: HOROZ     0.01384083   1.0000000      1.0000000      0.8600196 1.0000000
## Class: SEKER     1.00000000   0.8090438      0.4783635      1.0000000 0.4783635
## Class: SIRA      0.97468354   0.9085106      0.7189542      0.9933533 0.7189542
##                     Recall          F1 Prevalence Detection Rate
## Class: BARBUNYA 0.63888889 0.586326767 0.09705882    0.062009804
## Class: BOMBAY   0.00000000          NA 0.03823529    0.000000000
## Class: CALI     0.00204499 0.004081633 0.11985294    0.000245098
## Class: DERMASON 1.00000000 0.914408602 0.26053922    0.260539216
## Class: HOROZ    0.01384083 0.027303754 0.14166667    0.001960784
## Class: SEKER    1.00000000 0.647152741 0.14901961    0.149019608
## Class: SIRA     0.97468354 0.827512090 0.19362745    0.188725490
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.114460784         0.7903999
## Class: BOMBAY            0.000000000         0.5000000
## Class: CALI              0.000245098         0.5010225
## Class: DERMASON          0.309313725         0.9670202
## Class: HOROZ             0.001960784         0.5069204
## Class: SEKER             0.311519608         0.9045219
## Class: SIRA              0.262500000         0.9415971
db_tda_pc_5.50.5_n1_rf_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n1_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_rf_n1_3_fold<-(db_rf_fit_re-db_tda_pc_5.50.5_n1_rf_fit0_re)
diff_drybean_tda_pca_5.50.5_rf_n1_3_fold
##     Accuracy
## 1 0.01342308
## 2 0.01462130
## 3 0.01607300
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n1_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n1_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n1_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n1_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n1_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n1_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_rf.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n1_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_rf.n1_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1694
## 
## $winRight
## [1] 0.8306
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n1_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n1_3_fold
## $left
## [1] 0.0006398568
## 
## $rope
## [1] 0.01614588
## 
## $right
## [1] 0.9832143
# Rope Plot
plot(rope(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n1_3_fold),c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf_n1_3_fold))
#bf_tda_pca_5.50.5_rf.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_rf_n1_3_fold)
## t = 19.195, df = 2, p-value = 0.002703
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.01140940 0.01800219
## sample estimates:
##  mean of x 
## 0.01470579
### Test set diff
diff_drybean_tda_pca_5.50.5_rf.n1_test<-(db_rf_cf_ov_acc-db_tda_pc_5.50.5_n1_rf_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_rf.n1_test
##  Accuracy 
## 0.2553922
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n1_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n1_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n1_test$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n1_test$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n1_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_rf.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n1_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_rf.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1606333
## 
## $winRight
## [1] 0.8393667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n1_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_rf.n1_test)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf.n1_test)) #bf_tda_pca_5.50.5_rf.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n1_test))

##Node2

DryBean_TDA_PC_5.50.5_n2_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n2.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'), 
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry=  50 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 100 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 150 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 200 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 250 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 300 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 350 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 400 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 450 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 500 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 550 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 600 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 650 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 700 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 750 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 800 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 850 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 900 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry= 950 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold3: mtry=1000 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Can't have empty classes in y.
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo,
## : There were missing values in resampled performance measures.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_PC_5.50.5_n2_RfFit0
## Random Forest 
## 
## 9515 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6344, 6343, 6343 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.9009937  0.8750669
##    100  0.8997325  0.8734818
##    150  0.9020974  0.8764675
##    200  0.9009939  0.8750450
##    250  0.9006784  0.8746598
##    300  0.9013092  0.8754739
##    350  0.9005210  0.8744799
##    400  0.9000480  0.8738692
##    450  0.9008360  0.8748766
##    500  0.9009938  0.8750842
##    550  0.9011517  0.8752628
##    600  0.9014668  0.8756717
##    650  0.9008361  0.8748736
##    700  0.9002058  0.8740720
##    750  0.9009939  0.8750681
##    800  0.9013092  0.8754688
##    850  0.9013091  0.8754314
##    900  0.9003631  0.8742694
##    950  0.9011514  0.8752643
##   1000  0.9020974  0.8764672
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 150.
DryBean_TDA_PC_5.50.5_n2_RfFit0$resample
##    Accuracy    Kappa Resample
## 1 0.8984868 0.871988    Fold2
## 2 0.9057080 0.880947    Fold1
## 3        NA       NA    Fold3
db_tda_pc_5.50.5_n2_rf_fit0_re<-DryBean_TDA_PC_5.50.5_n2_RfFit0$resample[1]


summary(DryBean_TDA_PC_5.50.5_n2_RfFit0)
##                 Length Class      Mode     
## call                5  -none-     call     
## type                1  -none-     character
## predicted        9515  factor     numeric  
## err.rate         4000  -none-     numeric  
## confusion          56  -none-     numeric  
## votes           66605  matrix     numeric  
## oob.times        9515  -none-     numeric  
## classes             7  -none-     character
## importance         16  -none-     numeric  
## importanceSD        0  -none-     NULL     
## localImportance     0  -none-     NULL     
## proximity           0  -none-     NULL     
## ntree               1  -none-     numeric  
## mtry                1  -none-     numeric  
## forest             14  -none-     list     
## y                9515  factor     numeric  
## test                0  -none-     NULL     
## inbag               0  -none-     NULL     
## xNames             16  -none-     character
## problemType         1  -none-     character
## tuneValue           1  data.frame list     
## obsLevels           7  -none-     character
## param               1  -none-     list
vip(DryBean_TDA_PC_5.50.5_n2_RfFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n2_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_PC_5.50.5_n2_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n2_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n2_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
db_tda_pc_5.50.5_n2_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      394     27    2        0     3     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            1    128  472        0     7     0    0
##   DERMASON        0      0    0     1059     0   203    0
##   HOROZ           1      1   15        0   568     0    0
##   SEKER           0      0    0        0     0   376    0
##   SIRA            0      0    0        4     0    29  790
## 
## Overall Statistics
##                                          
##                Accuracy : 0.8968         
##                  95% CI : (0.8871, 0.906)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.8739         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.99495       0.00000      0.9652          0.9962
## Specificity                  0.99131       1.00000      0.9621          0.9327
## Pos Pred Value               0.92488           NaN      0.7763          0.8391
## Neg Pred Value               0.99945       0.96176      0.9951          0.9986
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.09657       0.00000      0.1157          0.2596
## Detection Prevalence         0.10441       0.00000      0.1490          0.3093
## Balanced Accuracy            0.99313       0.50000      0.9637          0.9645
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9827      0.61842      1.0000
## Specificity                0.9951      1.00000      0.9900
## Pos Pred Value             0.9709      1.00000      0.9599
## Neg Pred Value             0.9971      0.93737      1.0000
## Prevalence                 0.1417      0.14902      0.1936
## Detection Rate             0.1392      0.09216      0.1936
## Detection Prevalence       0.1434      0.09216      0.2017
## Balanced Accuracy          0.9889      0.80921      0.9950
db_tda_pc_5.50.5_n2_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      394     27    2        0     3     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            1    128  472        0     7     0    0
##   DERMASON        0      0    0     1059     0   203    0
##   HOROZ           1      1   15        0   568     0    0
##   SEKER           0      0    0        0     0   376    0
##   SIRA            0      0    0        4     0    29  790
## 
## Overall Statistics
##                                          
##                Accuracy : 0.8968         
##                  95% CI : (0.8871, 0.906)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.8739         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.99495       0.00000      0.9652          0.9962
## Specificity                  0.99131       1.00000      0.9621          0.9327
## Pos Pred Value               0.92488           NaN      0.7763          0.8391
## Neg Pred Value               0.99945       0.96176      0.9951          0.9986
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.09657       0.00000      0.1157          0.2596
## Detection Prevalence         0.10441       0.00000      0.1490          0.3093
## Balanced Accuracy            0.99313       0.50000      0.9637          0.9645
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9827      0.61842      1.0000
## Specificity                0.9951      1.00000      0.9900
## Pos Pred Value             0.9709      1.00000      0.9599
## Neg Pred Value             0.9971      0.93737      1.0000
## Prevalence                 0.1417      0.14902      0.1936
## Detection Rate             0.1392      0.09216      0.1936
## Detection Prevalence       0.1434      0.09216      0.2017
## Balanced Accuracy          0.9889      0.80921      0.9950
db_tda_pc_5.50.5_n2_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.8968137      0.8739038      0.8870710      0.9059839      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n2_rf_cf0_ov_acc<-db_tda_pc_5.50.5_n2_rf_cf0$overall[1]
db_tda_pc_5.50.5_n2_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9949495   0.9913138      0.9248826      0.9994527 0.9248826
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.9652352   0.9621275      0.7763158      0.9951037 0.7763158
## Class: DERMASON   0.9962371   0.9327146      0.8391442      0.9985806 0.8391442
## Class: HOROZ      0.9826990   0.9951456      0.9709402      0.9971388 0.9709402
## Class: SEKER      0.6184211   1.0000000      1.0000000      0.9373650 1.0000000
## Class: SIRA       1.0000000   0.9899696      0.9599028      1.0000000 0.9599028
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9949495 0.9586375 0.09705882     0.09656863
## Class: BOMBAY   0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.9652352 0.8605287 0.11985294     0.11568627
## Class: DERMASON 0.9962371 0.9109677 0.26053922     0.25955882
## Class: HOROZ    0.9826990 0.9767842 0.14166667     0.13921569
## Class: SEKER    0.6184211 0.7642276 0.14901961     0.09215686
## Class: SIRA     1.0000000 0.9795412 0.19362745     0.19362745
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.10441176         0.9931316
## Class: BOMBAY             0.00000000         0.5000000
## Class: CALI               0.14901961         0.9636814
## Class: DERMASON           0.30931373         0.9644758
## Class: HOROZ              0.14338235         0.9889223
## Class: SEKER              0.09215686         0.8092105
## Class: SIRA               0.20171569         0.9949848
db_tda_pc_5.50.5_n2_rf_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n2_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_rf_n2_3_fold<-(db_rf_fit_re-db_tda_pc_5.50.5_n2_rf_fit0_re)
diff_drybean_tda_pca_5.50.5_rf_n2_3_fold
##    Accuracy
## 1 0.0265998
## 2 0.0181435
## 3        NA
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n2_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n2_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n2_3_fold
## $probLeft
## [1] NA
## 
## $probRope
## [1] NA
## 
## $probRight
## [1] NA
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n2_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n2_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n2_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n2_3_fold_odds.left
## [1] NA
# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_rf.n2_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n2_3_fold),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_rf.n2_3_fold

# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n2_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n2_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n2_3_fold
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n2_3_fold),c(-0.01,0.01)))


#BayesFactor
#bf_tda_pca_5.50.5_rf.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf_n2_3_fold))
#bf_tda_pca_5.50.5_rf.n2_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n2_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_rf_n2_3_fold)
## t = 5.2911, df = 1, p-value = 0.1189
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.03135209  0.07609539
## sample estimates:
##  mean of x 
## 0.02237165
### Test set diff
diff_drybean_tda_pca_5.50.5_rf.n2_test<-(db_rf_cf_ov_acc-db_tda_pc_5.50.5_n2_rf_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_rf.n2_test
##   Accuracy 
## 0.02107843
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n2_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n2_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n2_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n2_test$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n2_test$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n2_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_rf.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n2_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_rf.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1619
## 
## $winRight
## [1] 0.8381
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n2_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n2_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n2_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n2_test),c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf.n2_test)) #bf_tda_pca_5.50.5_rf.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n2_test))

##Node3

DryBean_TDA_PC_5.50.5_n3_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n3.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'), 
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_PC_5.50.5_n3_RfFit0
## Random Forest 
## 
## 5355 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 3569, 3571, 3570 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.9363235  0.9111979
##    100  0.9381909  0.9137930
##    150  0.9372577  0.9125095
##    200  0.9370709  0.9122286
##    250  0.9380044  0.9135214
##    300  0.9380042  0.9135431
##    350  0.9361370  0.9109204
##    400  0.9374440  0.9127344
##    450  0.9374440  0.9127340
##    500  0.9380046  0.9134972
##    550  0.9381910  0.9138017
##    600  0.9381909  0.9137758
##    650  0.9381914  0.9137947
##    700  0.9365106  0.9114379
##    750  0.9368839  0.9119667
##    800  0.9389375  0.9148460
##    850  0.9380049  0.9135287
##    900  0.9380047  0.9135499
##    950  0.9378174  0.9132780
##   1000  0.9372577  0.9124974
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 800.
DryBean_TDA_PC_5.50.5_n3_RfFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9467489 0.9258106    Fold2
## 2 0.9361702 0.9108514    Fold1
## 3 0.9338936 0.9078760    Fold3
db_tda_pc_5.50.5_n3_rf_fit0_re<-DryBean_TDA_PC_5.50.5_n3_RfFit0$resample[1]


summary(DryBean_TDA_PC_5.50.5_n3_RfFit0)
##                 Length Class      Mode     
## call                5  -none-     call     
## type                1  -none-     character
## predicted        5355  factor     numeric  
## err.rate         4000  -none-     numeric  
## confusion          56  -none-     numeric  
## votes           37485  matrix     numeric  
## oob.times        5355  -none-     numeric  
## classes             7  -none-     character
## importance         16  -none-     numeric  
## importanceSD        0  -none-     NULL     
## localImportance     0  -none-     NULL     
## proximity           0  -none-     NULL     
## ntree               1  -none-     numeric  
## mtry                1  -none-     numeric  
## forest             14  -none-     list     
## y                5355  factor     numeric  
## test                0  -none-     NULL     
## inbag               0  -none-     NULL     
## xNames             16  -none-     character
## problemType         1  -none-     character
## tuneValue           1  data.frame list     
## obsLevels           7  -none-     character
## param               1  -none-     list
vip(DryBean_TDA_PC_5.50.5_n3_RfFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n3_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_PC_5.50.5_n3_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n3_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n3_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
db_tda_pc_5.50.5_n3_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      392      1    1       12     0   156   15
##   BOMBAY          0    104    0        0     0     0    0
##   CALI            0     51  488        0     0     0    1
##   DERMASON        0      0    0        1     0     0    0
##   HOROZ           0      0    0      829   576     3   49
##   SEKER           0      0    0        0     0     0    0
##   SIRA            4      0    0      221     2   449  725
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5603          
##                  95% CI : (0.5449, 0.5756)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.4841          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.98990       0.66667      0.9980       0.0009407
## Specificity                  0.94978       1.00000      0.9855       1.0000000
## Pos Pred Value               0.67938       1.00000      0.9037       1.0000000
## Neg Pred Value               0.99886       0.98692      0.9997       0.7396421
## Prevalence                   0.09706       0.03824      0.1199       0.2605392
## Detection Rate               0.09608       0.02549      0.1196       0.0002451
## Detection Prevalence         0.14142       0.02549      0.1324       0.0002451
## Balanced Accuracy            0.96984       0.83333      0.9917       0.5004704
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9965        0.000      0.9177
## Specificity                0.7484        1.000      0.7945
## Pos Pred Value             0.3953          NaN      0.5175
## Neg Pred Value             0.9992        0.851      0.9757
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1412        0.000      0.1777
## Detection Prevalence       0.3571        0.000      0.3434
## Balanced Accuracy          0.8725        0.500      0.8561
db_tda_pc_5.50.5_n3_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      392      1    1       12     0   156   15
##   BOMBAY          0    104    0        0     0     0    0
##   CALI            0     51  488        0     0     0    1
##   DERMASON        0      0    0        1     0     0    0
##   HOROZ           0      0    0      829   576     3   49
##   SEKER           0      0    0        0     0     0    0
##   SIRA            4      0    0      221     2   449  725
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5603          
##                  95% CI : (0.5449, 0.5756)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.4841          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.98990       0.66667      0.9980       0.0009407
## Specificity                  0.94978       1.00000      0.9855       1.0000000
## Pos Pred Value               0.67938       1.00000      0.9037       1.0000000
## Neg Pred Value               0.99886       0.98692      0.9997       0.7396421
## Prevalence                   0.09706       0.03824      0.1199       0.2605392
## Detection Rate               0.09608       0.02549      0.1196       0.0002451
## Detection Prevalence         0.14142       0.02549      0.1324       0.0002451
## Balanced Accuracy            0.96984       0.83333      0.9917       0.5004704
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9965        0.000      0.9177
## Specificity                0.7484        1.000      0.7945
## Pos Pred Value             0.3953          NaN      0.5175
## Neg Pred Value             0.9992        0.851      0.9757
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1412        0.000      0.1777
## Detection Prevalence       0.3571        0.000      0.3434
## Balanced Accuracy          0.8725        0.500      0.8561
db_tda_pc_5.50.5_n3_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.5602941      0.4840912      0.5449027      0.5755991      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n3_rf_cf0_ov_acc<-db_tda_pc_5.50.5_n3_rf_cf0$overall[1]
db_tda_pc_5.50.5_n3_rf_cf0$byClass
##                  Sensitivity Specificity Pos Pred Value Neg Pred Value
## Class: BARBUNYA 0.9898989899   0.9497828      0.6793761      0.9988581
## Class: BOMBAY   0.6666666667   1.0000000      1.0000000      0.9869215
## Class: CALI     0.9979550102   0.9855194      0.9037037      0.9997175
## Class: DERMASON 0.0009407338   1.0000000      1.0000000      0.7396421
## Class: HOROZ    0.9965397924   0.7484295      0.3953329      0.9992375
## Class: SEKER    0.0000000000   1.0000000            NaN      0.8509804
## Class: SIRA     0.9177215190   0.7945289      0.5174875      0.9757372
##                 Precision       Recall          F1 Prevalence Detection Rate
## Class: BARBUNYA 0.6793761 0.9898989899 0.805755396 0.09705882    0.096078431
## Class: BOMBAY   1.0000000 0.6666666667 0.800000000 0.03823529    0.025490196
## Class: CALI     0.9037037 0.9979550102 0.948493683 0.11985294    0.119607843
## Class: DERMASON 1.0000000 0.0009407338 0.001879699 0.26053922    0.000245098
## Class: HOROZ    0.3953329 0.9965397924 0.566093366 0.14166667    0.141176471
## Class: SEKER           NA 0.0000000000          NA 0.14901961    0.000000000
## Class: SIRA     0.5174875 0.9177215190 0.661798266 0.19362745    0.177696078
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.141421569         0.9698409
## Class: BOMBAY            0.025490196         0.8333333
## Class: CALI              0.132352941         0.9917372
## Class: DERMASON          0.000245098         0.5004704
## Class: HOROZ             0.357107843         0.8724846
## Class: SEKER             0.000000000         0.5000000
## Class: SIRA              0.343382353         0.8561252
db_tda_pc_5.50.5_n3_rf_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n3_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_rf_n3_3_fold<-(db_rf_fit_re-db_tda_pc_5.50.5_n3_rf_fit0_re)
diff_drybean_tda_pca_5.50.5_rf_n3_3_fold
##       Accuracy
## 1 -0.021662319
## 2 -0.012318734
## 3 -0.003163079
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n3_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n3_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n3_3_fold
## $probLeft
## [1] 0.5
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n3_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n3_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n3_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n3_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_rf.n3_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n3_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_rf.n3_3_fold
## $winLeft
## [1] 0.6055
## 
## $winRope
## [1] 0.3945
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n3_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n3_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n2_3_fold
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
plot(rope(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n3_3_fold),c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf_n3_3_fold))
#bf_tda_pca_5.50.5_rf.n3_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n3_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_rf_n3_3_fold)
## t = -2.3185, df = 2, p-value = 0.1463
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.03535910  0.01059635
## sample estimates:
##   mean of x 
## -0.01238138
### Test set diff
diff_drybean_tda_pca_5.50.5_rf.n3_test<-(db_rf_cf_ov_acc-db_tda_pc_5.50.5_n3_rf_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_rf.n3_test
## Accuracy 
## 0.357598
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n3_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n3_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n3_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n3_test$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n3_test$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n3_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_rf.n3_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n3_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_rf.n3_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1615
## 
## $winRight
## [1] 0.8385
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n3_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n3_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n3_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_rf.n3_test))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf.n3_test)) #bf_tda_pca_5.50.5_rf.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n2_test)

##Node4

DryBean_TDA_PC_5.50.5_n4_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n4.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'), 
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_PC_5.50.5_n4_RfFit0
## Random Forest 
## 
## 1590 samples
##   16 predictor
##    4 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'HOROZ' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1060, 1060, 1060 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.9704403  0.9569691
##    100  0.9691824  0.9551499
##    150  0.9691824  0.9551096
##    200  0.9710692  0.9578962
##    250  0.9698113  0.9560599
##    300  0.9698113  0.9560793
##    350  0.9691824  0.9551499
##    400  0.9698113  0.9560148
##    450  0.9685535  0.9542196
##    500  0.9685535  0.9542536
##    550  0.9691824  0.9551407
##    600  0.9691824  0.9551167
##    650  0.9710692  0.9579371
##    700  0.9691824  0.9551557
##    750  0.9691824  0.9551436
##    800  0.9691824  0.9551379
##    850  0.9710692  0.9578851
##    900  0.9691824  0.9551606
##    950  0.9691824  0.9551173
##   1000  0.9685535  0.9542314
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 200.
DryBean_TDA_PC_5.50.5_n4_RfFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9698113 0.9560312    Fold3
## 2 0.9698113 0.9562056    Fold2
## 3 0.9735849 0.9614517    Fold1
db_tda_pc_5.50.5_n4_rf_fit0_re<-DryBean_TDA_PC_5.50.5_n4_RfFit0$resample[1]


summary(DryBean_TDA_PC_5.50.5_n4_RfFit0)
##                 Length Class      Mode     
## call               5   -none-     call     
## type               1   -none-     character
## predicted       1590   factor     numeric  
## err.rate        2500   -none-     numeric  
## confusion         20   -none-     numeric  
## votes           6360   matrix     numeric  
## oob.times       1590   -none-     numeric  
## classes            4   -none-     character
## importance        16   -none-     numeric  
## importanceSD       0   -none-     NULL     
## localImportance    0   -none-     NULL     
## proximity          0   -none-     NULL     
## ntree              1   -none-     numeric  
## mtry               1   -none-     numeric  
## forest            14   -none-     list     
## y               1590   factor     numeric  
## test               0   -none-     NULL     
## inbag              0   -none-     NULL     
## xNames            16   -none-     character
## problemType        1   -none-     character
## tuneValue          1   data.frame list     
## obsLevels          4   -none-     character
## param              1   -none-     list
vip(DryBean_TDA_PC_5.50.5_n4_RfFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n4_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_PC_5.50.5_n4_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n4_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n4_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n4_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      272      0    4        0     0     4    0
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           73      0  469        0    13   102    9
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ          51      0   16     1063   565   502  781
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3583          
##                  95% CI : (0.3436, 0.3733)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2615          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.68687       1.00000      0.9591          0.0000
## Specificity                  0.99783       1.00000      0.9451          1.0000
## Pos Pred Value               0.97143       1.00000      0.7042             NaN
## Neg Pred Value               0.96737       1.00000      0.9941          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.06667       0.03824      0.1150          0.0000
## Detection Prevalence         0.06863       0.03824      0.1632          0.0000
## Balanced Accuracy            0.84235       1.00000      0.9521          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9775        0.000      0.0000
## Specificity                0.3110        1.000      1.0000
## Pos Pred Value             0.1897          NaN         NaN
## Neg Pred Value             0.9882        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1385        0.000      0.0000
## Detection Prevalence       0.7299        0.000      0.0000
## Balanced Accuracy          0.6442        0.500      0.5000
db_tda_pc_5.50.5_n4_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      272      0    4        0     0     4    0
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           73      0  469        0    13   102    9
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ          51      0   16     1063   565   502  781
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3583          
##                  95% CI : (0.3436, 0.3733)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2615          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.68687       1.00000      0.9591          0.0000
## Specificity                  0.99783       1.00000      0.9451          1.0000
## Pos Pred Value               0.97143       1.00000      0.7042             NaN
## Neg Pred Value               0.96737       1.00000      0.9941          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.06667       0.03824      0.1150          0.0000
## Detection Prevalence         0.06863       0.03824      0.1632          0.0000
## Balanced Accuracy            0.84235       1.00000      0.9521          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9775        0.000      0.0000
## Specificity                0.3110        1.000      1.0000
## Pos Pred Value             0.1897          NaN         NaN
## Neg Pred Value             0.9882        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1385        0.000      0.0000
## Detection Prevalence       0.7299        0.000      0.0000
## Balanced Accuracy          0.6442        0.500      0.5000
db_tda_pc_5.50.5_n4_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   3.583333e-01   2.615270e-01   3.436031e-01   3.732667e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##   2.937776e-43            NaN
db_tda_pc_5.50.5_n4_rf_cf0_ov_acc<-db_tda_pc_5.50.5_n4_rf_cf0$overall[1]
db_tda_pc_5.50.5_n4_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.6868687   0.9978284      0.9714286      0.9673684 0.9714286
## Class: BOMBAY     1.0000000   1.0000000      1.0000000      1.0000000 1.0000000
## Class: CALI       0.9591002   0.9451406      0.7042042      0.9941418 0.7042042
## Class: DERMASON   0.0000000   1.0000000            NaN      0.7394608        NA
## Class: HOROZ      0.9775087   0.3109652      0.1897246      0.9882033 0.1897246
## Class: SEKER      0.0000000   1.0000000            NaN      0.8509804        NA
## Class: SIRA       0.0000000   1.0000000            NaN      0.8063725        NA
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.6868687 0.8047337 0.09705882     0.06666667
## Class: BOMBAY   1.0000000 1.0000000 0.03823529     0.03823529
## Class: CALI     0.9591002 0.8121212 0.11985294     0.11495098
## Class: DERMASON 0.0000000        NA 0.26053922     0.00000000
## Class: HOROZ    0.9775087 0.3177728 0.14166667     0.13848039
## Class: SEKER    0.0000000        NA 0.14901961     0.00000000
## Class: SIRA     0.0000000        NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.06862745         0.8423486
## Class: BOMBAY             0.03823529         1.0000000
## Class: CALI               0.16323529         0.9521204
## Class: DERMASON           0.00000000         0.5000000
## Class: HOROZ              0.72990196         0.6442369
## Class: SEKER              0.00000000         0.5000000
## Class: SIRA               0.00000000         0.5000000
db_tda_pc_5.50.5_n4_rf_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n4_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_rf_n4_3_fold<-(db_rf_fit_re-db_tda_pc_5.50.5_n4_rf_fit0_re)
diff_drybean_tda_pca_5.50.5_rf_n4_3_fold
##      Accuracy
## 1 -0.04472476
## 2 -0.04595984
## 3 -0.04285443
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n4_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n4_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n4_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n4_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n4_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n4_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_rf.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n4_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_rf.n4_3_fold
## $winLeft
## [1] 0.9906
## 
## $winRope
## [1] 0.0094
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n4_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n4_3_fold
## $left
## [1] 0.9995446
## 
## $rope
## [1] 0.0002727294
## 
## $right
## [1] 0.0001827024
# Rope Plot
plot(rope(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n4_3_fold),c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf_n4_3_fold))
#bf_tda_pca_5.50.5_rf.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_rf_n4_3_fold)
## t = -49.312, df = 2, p-value = 0.000411
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.04839696 -0.04062906
## sample estimates:
##   mean of x 
## -0.04451301
### Test set diff
diff_drybean_tda_pca_5.50.5_rf.n4_test<-(db_rf_cf_ov_acc-db_tda_pc_5.50.5_n4_rf_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_rf.n4_test
##  Accuracy 
## 0.5595588
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n4_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n4_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n4_test$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n4_test$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_rf.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n4_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_rf.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1571333
## 
## $winRight
## [1] 0.8428667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n4_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_rf.n4_test))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf.n4_test)) #bf_tda_pca_5.50.5_rf.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n4_test))

##Node5

DryBean_TDA_PC_5.50.5_n5_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n5.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'), 
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning: model fit failed for Fold2: mtry=  50 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 100 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 150 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 200 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 250 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 300 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 350 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 400 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 450 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 500 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 550 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 600 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 650 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 700 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 750 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 800 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 850 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 900 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry= 950 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning: model fit failed for Fold2: mtry=1000 Error in randomForest.default(x, y, mtry = param$mtry, ...) : 
##   Need at least two classes to do classification.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo,
## : There were missing values in resampled performance measures.
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_PC_5.50.5_n5_RfFit0
## Random Forest 
## 
## 417 samples
##  16 predictor
##   2 classes: 'BOMBAY', 'CALI' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 278, 278, 278 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy  Kappa
##     50  1         NaN  
##    100  1         NaN  
##    150  1         NaN  
##    200  1         NaN  
##    250  1         NaN  
##    300  1         NaN  
##    350  1         NaN  
##    400  1         NaN  
##    450  1         NaN  
##    500  1         NaN  
##    550  1         NaN  
##    600  1         NaN  
##    650  1         NaN  
##    700  1         NaN  
##    750  1         NaN  
##    800  1         NaN  
##    850  1         NaN  
##    900  1         NaN  
##    950  1         NaN  
##   1000  1         NaN  
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 50.
DryBean_TDA_PC_5.50.5_n5_RfFit0$resample
##   Accuracy Kappa Resample
## 1        1    NA    Fold1
## 2        1    NA    Fold3
## 3       NA    NA    Fold2
db_tda_pc_5.50.5_n5_rf_fit0_re<-DryBean_TDA_PC_5.50.5_n5_RfFit0$resample[1]


summary(DryBean_TDA_PC_5.50.5_n5_RfFit0)
##                 Length Class      Mode     
## call               5   -none-     call     
## type               1   -none-     character
## predicted        417   factor     numeric  
## err.rate        1500   -none-     numeric  
## confusion          6   -none-     numeric  
## votes            834   matrix     numeric  
## oob.times        417   -none-     numeric  
## classes            2   -none-     character
## importance        16   -none-     numeric  
## importanceSD       0   -none-     NULL     
## localImportance    0   -none-     NULL     
## proximity          0   -none-     NULL     
## ntree              1   -none-     numeric  
## mtry               1   -none-     numeric  
## forest            14   -none-     list     
## y                417   factor     numeric  
## test               0   -none-     NULL     
## inbag              0   -none-     NULL     
## xNames            16   -none-     character
## problemType        1   -none-     character
## tuneValue          1   data.frame list     
## obsLevels          2   -none-     character
## param              1   -none-     list
vip(DryBean_TDA_PC_5.50.5_n5_RfFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n5_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_PC_5.50.5_n5_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n5_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n5_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n5_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          1    156    0        0     0     0    0
##   CALI          395      0  489     1063   578   608  790
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                          
##                Accuracy : 0.1581         
##                  95% CI : (0.147, 0.1696)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : 1              
##                                          
##                   Kappa : 0.0468         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000     1.00000          0.0000
## Specificity                  1.00000       0.99975     0.04372          1.0000
## Pos Pred Value                   NaN       0.99363     0.12465             NaN
## Neg Pred Value               0.90294       1.00000     1.00000          0.7395
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.00000       0.03824     0.11985          0.0000
## Detection Prevalence         0.00000       0.03848     0.96152          0.0000
## Balanced Accuracy            0.50000       0.99987     0.52186          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n5_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          1    156    0        0     0     0    0
##   CALI          395      0  489     1063   578   608  790
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                          
##                Accuracy : 0.1581         
##                  95% CI : (0.147, 0.1696)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : 1              
##                                          
##                   Kappa : 0.0468         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000     1.00000          0.0000
## Specificity                  1.00000       0.99975     0.04372          1.0000
## Pos Pred Value                   NaN       0.99363     0.12465             NaN
## Neg Pred Value               0.90294       1.00000     1.00000          0.7395
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.00000       0.03824     0.11985          0.0000
## Detection Prevalence         0.00000       0.03848     0.96152          0.0000
## Balanced Accuracy            0.50000       0.99987     0.52186          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n5_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##     0.15808824     0.04684314     0.14701909     0.16964931     0.26053922 
## AccuracyPValue  McnemarPValue 
##     1.00000000            NaN
db_tda_pc_5.50.5_n5_rf_cf0_ov_acc<-db_tda_pc_5.50.5_n5_rf_cf0$overall[1]
db_tda_pc_5.50.5_n5_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA           0  1.00000000            NaN      0.9029412        NA
## Class: BOMBAY             1  0.99974516      0.9936306      1.0000000 0.9936306
## Class: CALI               1  0.04372041      0.1246495      1.0000000 0.1246495
## Class: DERMASON           0  1.00000000            NaN      0.7394608        NA
## Class: HOROZ              0  1.00000000            NaN      0.8583333        NA
## Class: SEKER              0  1.00000000            NaN      0.8509804        NA
## Class: SIRA               0  1.00000000            NaN      0.8063725        NA
##                 Recall        F1 Prevalence Detection Rate Detection Prevalence
## Class: BARBUNYA      0        NA 0.09705882     0.00000000           0.00000000
## Class: BOMBAY        1 0.9968051 0.03823529     0.03823529           0.03848039
## Class: CALI          1 0.2216682 0.11985294     0.11985294           0.96151961
## Class: DERMASON      0        NA 0.26053922     0.00000000           0.00000000
## Class: HOROZ         0        NA 0.14166667     0.00000000           0.00000000
## Class: SEKER         0        NA 0.14901961     0.00000000           0.00000000
## Class: SIRA          0        NA 0.19362745     0.00000000           0.00000000
##                 Balanced Accuracy
## Class: BARBUNYA         0.5000000
## Class: BOMBAY           0.9998726
## Class: CALI             0.5218602
## Class: DERMASON         0.5000000
## Class: HOROZ            0.5000000
## Class: SEKER            0.5000000
## Class: SIRA             0.5000000
db_tda_pc_5.50.5_n5_rf_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n5_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_rf_n5_3_fold<-(db_rf_fit_re-db_tda_pc_5.50.5_n5_rf_fit0_re)
diff_drybean_tda_pca_5.50.5_rf_n5_3_fold
##      Accuracy
## 1 -0.07491344
## 2 -0.07614852
## 3          NA
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n5_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n5_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n5_3_fold
## $probLeft
## [1] NA
## 
## $probRope
## [1] NA
## 
## $probRight
## [1] NA
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n5_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n5_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n5_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n5_3_fold_odds.left
## [1] NA
# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_rf.n5_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n5_3_fold),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_rf.n5_3_fold

# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n5_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n5_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n5_3_fold
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_rf_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf_n5_3_fold))
#bf_tda_pca_5.50.5_rf.n5_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf_n5_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_rf_n5_3_fold)
## t = -122.31, df = 1, p-value = 0.005205
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.08337758 -0.06768439
## sample estimates:
##   mean of x 
## -0.07553098
### Test set diff
diff_drybean_tda_pca_5.50.5_rf.n5_test<-(db_rf_cf_ov_acc-db_tda_pc_5.50.5_n5_rf_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_rf.n5_test
##  Accuracy 
## 0.7598039
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_rf.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n5_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_rf.n5_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_rf.n5_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_rf.n5_test$probLeft/bst_dbf_db_tda_pca_5.50.5_rf.n5_test$probRight
bst_dbf_db_tda_pca_5.50.5_rf.n5_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_rf.n5_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n5_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_rf.n5_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1565333
## 
## $winRight
## [1] 0.8434667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_rf.n5_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n5_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_rf.n5_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_rf.n5_test))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_rf.n5_test)) #bf_tda_pca_5.50.5_rf.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_rf.n5_test))

##With TDA KDE filter 5 intervals, 50% overlap, 5 bins 
##Node1

DryBean_TDA_KDE_5.50.5_n1_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n1.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'), 
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_KDE_5.50.5_n1_RfFit0
## Random Forest 
## 
## 8473 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5649, 5648, 5649 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.9512562  0.9417543
##    100  0.9511381  0.9416175
##    150  0.9507841  0.9411931
##    200  0.9512562  0.9417534
##    250  0.9505481  0.9409065
##    300  0.9511382  0.9416147
##    350  0.9509022  0.9413309
##    400  0.9509022  0.9413307
##    450  0.9503121  0.9406302
##    500  0.9507843  0.9411942
##    550  0.9512562  0.9417537
##    600  0.9514924  0.9420387
##    650  0.9505481  0.9409120
##    700  0.9513743  0.9418963
##    750  0.9512564  0.9417572
##    800  0.9510201  0.9414767
##    850  0.9507843  0.9411931
##    900  0.9512564  0.9417549
##    950  0.9511382  0.9416140
##   1000  0.9510202  0.9414757
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 600.
DryBean_TDA_KDE_5.50.5_n1_RfFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9564602 0.9479760    Fold2
## 2 0.9514873 0.9420608    Fold1
## 3 0.9465297 0.9360791    Fold3
ad_tda_kde_5.50.5_n1_rf_fit0_re<-DryBean_TDA_KDE_5.50.5_n1_RfFit0$resample[1]


summary(DryBean_TDA_KDE_5.50.5_n1_RfFit0)
##                 Length Class      Mode     
## call                5  -none-     call     
## type                1  -none-     character
## predicted        8473  factor     numeric  
## err.rate         4000  -none-     numeric  
## confusion          56  -none-     numeric  
## votes           59311  matrix     numeric  
## oob.times        8473  -none-     numeric  
## classes             7  -none-     character
## importance         16  -none-     numeric  
## importanceSD        0  -none-     NULL     
## localImportance     0  -none-     NULL     
## proximity           0  -none-     NULL     
## ntree               1  -none-     numeric  
## mtry                1  -none-     numeric  
## forest             14  -none-     list     
## y                8473  factor     numeric  
## test                0  -none-     NULL     
## inbag               0  -none-     NULL     
## xNames             16  -none-     character
## problemType         1  -none-     character
## tuneValue           1  data.frame list     
## obsLevels           7  -none-     character
## param               1  -none-     list
vip(DryBean_TDA_KDE_5.50.5_n1_RfFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n1_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n1_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n1_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n1_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
ad_tda_kde_5.50.5_n1_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      394      0    0        1     0     0    1
##   BOMBAY          0    156    0        0     0     0    0
##   CALI            0      0  488        0     0     0    0
##   DERMASON        1      0    0      881     6     3  119
##   HOROZ           0      0    0        1   572     0    0
##   SEKER           0      0    0       32     0   589   11
##   SIRA            1      0    1      148     0    16  659
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9164          
##                  95% CI : (0.9075, 0.9247)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8991          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.99495       1.00000      0.9980          0.8288
## Specificity                  0.99946       1.00000      1.0000          0.9572
## Pos Pred Value               0.99495       1.00000      1.0000          0.8723
## Neg Pred Value               0.99946       1.00000      0.9997          0.9407
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.09657       0.03824      0.1196          0.2159
## Detection Prevalence         0.09706       0.03824      0.1196          0.2475
## Balanced Accuracy            0.99720       1.00000      0.9990          0.8930
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9896       0.9688      0.8342
## Specificity                0.9997       0.9876      0.9495
## Pos Pred Value             0.9983       0.9320      0.7988
## Neg Pred Value             0.9983       0.9945      0.9598
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1402       0.1444      0.1615
## Detection Prevalence       0.1404       0.1549      0.2022
## Balanced Accuracy          0.9947       0.9782      0.8919
ad_tda_kde_5.50.5_n1_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      394      0    0        1     0     0    1
##   BOMBAY          0    156    0        0     0     0    0
##   CALI            0      0  488        0     0     0    0
##   DERMASON        1      0    0      881     6     3  119
##   HOROZ           0      0    0        1   572     0    0
##   SEKER           0      0    0       32     0   589   11
##   SIRA            1      0    1      148     0    16  659
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9164          
##                  95% CI : (0.9075, 0.9247)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8991          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.99495       1.00000      0.9980          0.8288
## Specificity                  0.99946       1.00000      1.0000          0.9572
## Pos Pred Value               0.99495       1.00000      1.0000          0.8723
## Neg Pred Value               0.99946       1.00000      0.9997          0.9407
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.09657       0.03824      0.1196          0.2159
## Detection Prevalence         0.09706       0.03824      0.1196          0.2475
## Balanced Accuracy            0.99720       1.00000      0.9990          0.8930
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9896       0.9688      0.8342
## Specificity                0.9997       0.9876      0.9495
## Pos Pred Value             0.9983       0.9320      0.7988
## Neg Pred Value             0.9983       0.9945      0.9598
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1402       0.1444      0.1615
## Detection Prevalence       0.1404       0.1549      0.2022
## Balanced Accuracy          0.9947       0.9782      0.8919
ad_tda_kde_5.50.5_n1_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.9164216      0.8990787      0.9075056      0.9247357      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
ad_tda_kde_5.50.5_n1_rf_cf0_ov_acc<-ad_tda_kde_5.50.5_n1_rf_cf0$overall[1]
ad_tda_kde_5.50.5_n1_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9949495   0.9994571      0.9949495      0.9994571 0.9949495
## Class: BOMBAY     1.0000000   1.0000000      1.0000000      1.0000000 1.0000000
## Class: CALI       0.9979550   1.0000000      1.0000000      0.9997216 1.0000000
## Class: DERMASON   0.8287865   0.9572423      0.8722772      0.9407166 0.8722772
## Class: HOROZ      0.9896194   0.9997144      0.9982548      0.9982891 0.9982548
## Class: SEKER      0.9687500   0.9876152      0.9319620      0.9944896 0.9319620
## Class: SIRA       0.8341772   0.9495441      0.7987879      0.9597542 0.7987879
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9949495 0.9949495 0.09705882     0.09656863
## Class: BOMBAY   1.0000000 1.0000000 0.03823529     0.03823529
## Class: CALI     0.9979550 0.9989765 0.11985294     0.11960784
## Class: DERMASON 0.8287865 0.8499759 0.26053922     0.21593137
## Class: HOROZ    0.9896194 0.9939183 0.14166667     0.14019608
## Class: SEKER    0.9687500 0.9500000 0.14901961     0.14436275
## Class: SIRA     0.8341772 0.8160991 0.19362745     0.16151961
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09705882         0.9972033
## Class: BOMBAY             0.03823529         1.0000000
## Class: CALI               0.11960784         0.9989775
## Class: DERMASON           0.24754902         0.8930144
## Class: HOROZ              0.14044118         0.9946669
## Class: SEKER              0.15490196         0.9781826
## Class: SIRA               0.20220588         0.8918606
ad_tda_kde_5.50.5_n1_rf_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n1_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_rf_n1_3_fold<-(db_rf_fit_re-ad_tda_kde_5.50.5_n1_rf_fit0_re)
diff_drybean_tda_kde_5.50.5_rf_n1_3_fold
##      Accuracy
## 1 -0.03137362
## 2 -0.02763577
## 3 -0.01579927
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n1_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n1_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n1_3_fold_odds.left<-bst_tda_kde_5.50.5_rf.n1_3_fold$probLeft/bst_tda_kde_5.50.5_rf.n1_3_fold$probRight
bst_tda_kde_5.50.5_rf.n1_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n1_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n1_3_fold
## $winLeft
## [1] 0.9629333
## 
## $winRope
## [1] 0.03706667
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n1_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n1_3_fold
## $left
## [1] 0.9448367
## 
## $rope
## [1] 0.04354533
## 
## $right
## [1] 0.01161796
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_rf_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf_n1_3_fold))
#bf_tda_kde_5.50.5_rf.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_rf_n1_3_fold)
## t = -5.3122, df = 2, p-value = 0.03366
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.04513362 -0.00473882
## sample estimates:
##   mean of x 
## -0.02493622
### Test set diff
diff_drybean_tda_kde_5.50.5_rf.n1_test<-(db_rf_cf_ov_acc-ad_tda_kde_5.50.5_n1_rf_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_rf.n1_test
##    Accuracy 
## 0.001470588
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n1_test),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 1
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n1_test_odds.left<-bst_tda_kde_5.50.5_rf.n1_test$probLeft/bst_tda_kde_5.50.5_rf.n1_test$probRight
bst_tda_kde_5.50.5_rf.n1_test_odds.left
## [1] NaN
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n1_test),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 1
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n1_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_rf.n1_test))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf.n1_test)) #bf_tda_kde_5.50.5_rf.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n1_test))

##Node2

DryBean_TDA_KDE_5.50.5_n2_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n2.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'),                    
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_KDE_5.50.5_n2_RfFit0
## Random Forest 
## 
## 7582 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5054, 5055, 5055 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.9444737  0.9290609
##    100  0.9422315  0.9261804
##    150  0.9436822  0.9280384
##    200  0.9430229  0.9271944
##    250  0.9435506  0.9278785
##    300  0.9430231  0.9271971
##    350  0.9431549  0.9273653
##    400  0.9435505  0.9278756
##    450  0.9419678  0.9258586
##    500  0.9430229  0.9271926
##    550  0.9424954  0.9265167
##    600  0.9430228  0.9272057
##    650  0.9428913  0.9270361
##    700  0.9419678  0.9258475
##    750  0.9432868  0.9275378
##    800  0.9423634  0.9263604
##    850  0.9424954  0.9265269
##    900  0.9439459  0.9283897
##    950  0.9420998  0.9260186
##   1000  0.9418359  0.9256763
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 50.
DryBean_TDA_KDE_5.50.5_n2_RfFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9450158 0.9298876    Fold1
## 2 0.9457855 0.9306419    Fold3
## 3 0.9426197 0.9266532    Fold2
ad_tda_KDE_5.50.5_n2_rf_fit0_re<-DryBean_TDA_KDE_5.50.5_n2_RfFit0$resample[1]


summary(DryBean_TDA_KDE_5.50.5_n2_RfFit0)
##                 Length Class      Mode     
## call                5  -none-     call     
## type                1  -none-     character
## predicted        7582  factor     numeric  
## err.rate         3500  -none-     numeric  
## confusion          42  -none-     numeric  
## votes           45492  matrix     numeric  
## oob.times        7582  -none-     numeric  
## classes             6  -none-     character
## importance         16  -none-     numeric  
## importanceSD        0  -none-     NULL     
## localImportance     0  -none-     NULL     
## proximity           0  -none-     NULL     
## ntree               1  -none-     numeric  
## mtry                1  -none-     numeric  
## forest             14  -none-     list     
## y                7582  factor     numeric  
## test                0  -none-     NULL     
## inbag               0  -none-     NULL     
## xNames             16  -none-     character
## problemType         1  -none-     character
## tuneValue           1  data.frame list     
## obsLevels           6  -none-     character
## param               1  -none-     list
vip(DryBean_TDA_KDE_5.50.5_n2_RfFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n2_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n2_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n2_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n2_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
ad_tda_kde_5.50.5_n2_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      340      1    5        0     3     1    1
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           10    150  475        0    13     0    3
##   DERMASON        0      0    0     1007     3     9   96
##   HOROZ           3      0    6        1   558     0    1
##   SEKER           7      0    0       23     0   593    3
##   SIRA           36      5    3       32     1     5  686
## 
## Overall Statistics
##                                          
##                Accuracy : 0.8968         
##                  95% CI : (0.8871, 0.906)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.8745         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.85859       0.00000      0.9714          0.9473
## Specificity                  0.99701       1.00000      0.9510          0.9642
## Pos Pred Value               0.96866           NaN      0.7296          0.9031
## Neg Pred Value               0.98498       0.96176      0.9959          0.9811
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08333       0.00000      0.1164          0.2468
## Detection Prevalence         0.08603       0.00000      0.1596          0.2733
## Balanced Accuracy            0.92780       0.50000      0.9612          0.9558
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9654       0.9753      0.8684
## Specificity                0.9969       0.9905      0.9751
## Pos Pred Value             0.9807       0.9473      0.8932
## Neg Pred Value             0.9943       0.9957      0.9686
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1368       0.1453      0.1681
## Detection Prevalence       0.1395       0.1534      0.1882
## Balanced Accuracy          0.9811       0.9829      0.9217
ad_tda_kde_5.50.5_n2_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      340      1    5        0     3     1    1
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           10    150  475        0    13     0    3
##   DERMASON        0      0    0     1007     3     9   96
##   HOROZ           3      0    6        1   558     0    1
##   SEKER           7      0    0       23     0   593    3
##   SIRA           36      5    3       32     1     5  686
## 
## Overall Statistics
##                                          
##                Accuracy : 0.8968         
##                  95% CI : (0.8871, 0.906)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.8745         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.85859       0.00000      0.9714          0.9473
## Specificity                  0.99701       1.00000      0.9510          0.9642
## Pos Pred Value               0.96866           NaN      0.7296          0.9031
## Neg Pred Value               0.98498       0.96176      0.9959          0.9811
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08333       0.00000      0.1164          0.2468
## Detection Prevalence         0.08603       0.00000      0.1596          0.2733
## Balanced Accuracy            0.92780       0.50000      0.9612          0.9558
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9654       0.9753      0.8684
## Specificity                0.9969       0.9905      0.9751
## Pos Pred Value             0.9807       0.9473      0.8932
## Neg Pred Value             0.9943       0.9957      0.9686
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1368       0.1453      0.1681
## Detection Prevalence       0.1395       0.1534      0.1882
## Balanced Accuracy          0.9811       0.9829      0.9217
ad_tda_kde_5.50.5_n2_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.8968137      0.8745084      0.8870710      0.9059839      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
ad_tda_kde_5.50.5_n2_rf_cf0_ov_acc<-ad_tda_kde_5.50.5_n2_rf_cf0$overall[1]
ad_tda_kde_5.50.5_n2_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.8585859   0.9970141      0.9686610      0.9849826 0.9686610
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.9713701   0.9509886      0.7296467      0.9959172 0.7296467
## Class: DERMASON   0.9473189   0.9642029      0.9031390      0.9811130 0.9031390
## Class: HOROZ      0.9653979   0.9968589      0.9806678      0.9943036 0.9806678
## Class: SEKER      0.9753289   0.9904954      0.9472843      0.9956572 0.9472843
## Class: SIRA       0.8683544   0.9750760      0.8932292      0.9685990 0.8932292
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.8585859 0.9103079 0.09705882     0.08333333
## Class: BOMBAY   0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.9713701 0.8333333 0.11985294     0.11642157
## Class: DERMASON 0.9473189 0.9247016 0.26053922     0.24681373
## Class: HOROZ    0.9653979 0.9729730 0.14166667     0.13676471
## Class: SEKER    0.9753289 0.9611021 0.14901961     0.14534314
## Class: SIRA     0.8683544 0.8806162 0.19362745     0.16813725
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.08602941         0.9278000
## Class: BOMBAY             0.00000000         0.5000000
## Class: CALI               0.15955882         0.9611794
## Class: DERMASON           0.27328431         0.9557609
## Class: HOROZ              0.13946078         0.9811284
## Class: SEKER              0.15343137         0.9829122
## Class: SIRA               0.18823529         0.9217152
ad_tda_kde_5.50.5_n2_rf_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n2_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_rf_n2_3_fold<-(db_rf_fit_re-ad_tda_KDE_5.50.5_n2_rf_fit0_re)
diff_drybean_tda_kde_5.50.5_rf_n2_3_fold
##      Accuracy
## 1 -0.01992926
## 2 -0.02193404
## 3 -0.01188923
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n2_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n2_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n2_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n2_3_fold_odds.left<-bst_tda_kde_5.50.5_rf.n2_3_fold$probLeft/bst_tda_kde_5.50.5_rf.n2_3_fold$probRight
bst_tda_kde_5.50.5_rf.n2_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n2_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n2_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n2_3_fold
## $winLeft
## [1] 0.9129667
## 
## $winRope
## [1] 0.08703333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n2_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n2_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n2_3_fold
## $left
## [1] 0.9224685
## 
## $rope
## [1] 0.0696636
## 
## $right
## [1] 0.007867899
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_rf_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf_n2_3_fold))
#bf_tda_kde_5.50.5_rf.n2_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n2_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_rf_n2_3_fold)
## t = -5.8378, df = 2, p-value = 0.02811
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.031123199 -0.004711821
## sample estimates:
##   mean of x 
## -0.01791751
### Test set diff
diff_drybean_tda_kde_5.50.5_rf.n2_test<-(db_rf_cf_ov_acc-ad_tda_kde_5.50.5_n2_rf_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_rf.n2_test
##   Accuracy 
## 0.02107843
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n2_test),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n2_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n2_test_odds.left<-bst_tda_kde_5.50.5_rf.n2_test$probLeft/bst_tda_kde_5.50.5_rf.n2_test$probRight
bst_tda_kde_5.50.5_rf.n2_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n2_test),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1580667
## 
## $winRight
## [1] 0.8419333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n2_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n2_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n2_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_rf.n2_test))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf.n2_test)) #bf_tda_kde_5.50.5_rf.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n2_test))

##Node3

DryBean_TDA_KDE_5.50.5_n3_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n3.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'),                    
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_KDE_5.50.5_n3_RfFit0
## Random Forest 
## 
## 4149 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 2766, 2766, 2766 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.9103398  0.8647137
##    100  0.9117860  0.8669165
##    150  0.9113039  0.8661831
##    200  0.9103398  0.8647139
##    250  0.9132321  0.8690892
##    300  0.9108219  0.8654454
##    350  0.9108219  0.8654660
##    400  0.9127501  0.8683567
##    450  0.9110629  0.8658288
##    500  0.9117860  0.8669082
##    550  0.9113039  0.8661768
##    600  0.9108219  0.8654641
##    650  0.9113039  0.8661911
##    700  0.9125090  0.8680099
##    750  0.9105809  0.8650758
##    800  0.9129911  0.8687300
##    850  0.9113039  0.8661818
##    900  0.9108219  0.8654443
##    950  0.9122680  0.8676217
##   1000  0.9110629  0.8658204
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 250.
DryBean_TDA_KDE_5.50.5_n3_RfFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9096168 0.8638074    Fold1
## 2 0.9146782 0.8709429    Fold3
## 3 0.9154013 0.8725173    Fold2
ad_tda_kde_5.50.5_n3_rf_fit0_re<-DryBean_TDA_KDE_5.50.5_n3_RfFit0$resample[1]


summary(DryBean_TDA_KDE_5.50.5_n3_RfFit0)
##                 Length Class      Mode     
## call                5  -none-     call     
## type                1  -none-     character
## predicted        4149  factor     numeric  
## err.rate         3500  -none-     numeric  
## confusion          42  -none-     numeric  
## votes           24894  matrix     numeric  
## oob.times        4149  -none-     numeric  
## classes             6  -none-     character
## importance         16  -none-     numeric  
## importanceSD        0  -none-     NULL     
## localImportance     0  -none-     NULL     
## proximity           0  -none-     NULL     
## ntree               1  -none-     numeric  
## mtry                1  -none-     numeric  
## forest             14  -none-     list     
## y                4149  factor     numeric  
## test                0  -none-     NULL     
## inbag               0  -none-     NULL     
## xNames             16  -none-     character
## problemType         1  -none-     character
## tuneValue           1  data.frame list     
## obsLevels           6  -none-     character
## param               1  -none-     list
vip(DryBean_TDA_KDE_5.50.5_n3_RfFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n3_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n3_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n3_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n3_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
ad_tda_kde_5.50.5_n3_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      297     64  129        0     9     1   15
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    1        0     0     0    0
##   DERMASON        0      0    0     1032     6     5   61
##   HOROZ          22     18  259        0   553     0    6
##   SEKER          16      4    1       15     0   588    3
##   SIRA           61     70   99       16    10    14  705
## 
## Overall Statistics
##                                           
##                Accuracy : 0.7784          
##                  95% CI : (0.7654, 0.7911)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.7292          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.75000       0.00000   0.0020450          0.9708
## Specificity                  0.94083       1.00000   1.0000000          0.9761
## Pos Pred Value               0.57670           NaN   1.0000000          0.9348
## Neg Pred Value               0.97223       0.96176   0.8803628          0.9896
## Prevalence                   0.09706       0.03824   0.1198529          0.2605
## Detection Rate               0.07279       0.00000   0.0002451          0.2529
## Detection Prevalence         0.12623       0.00000   0.0002451          0.2706
## Balanced Accuracy            0.84541       0.50000   0.5010225          0.9735
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9567       0.9671      0.8924
## Specificity                0.9129       0.9888      0.9179
## Pos Pred Value             0.6445       0.9378      0.7231
## Neg Pred Value             0.9922       0.9942      0.9726
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1355       0.1441      0.1728
## Detection Prevalence       0.2103       0.1537      0.2390
## Balanced Accuracy          0.9348       0.9779      0.9052
ad_tda_kde_5.50.5_n3_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      297     64  129        0     9     1   15
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    1        0     0     0    0
##   DERMASON        0      0    0     1032     6     5   61
##   HOROZ          22     18  259        0   553     0    6
##   SEKER          16      4    1       15     0   588    3
##   SIRA           61     70   99       16    10    14  705
## 
## Overall Statistics
##                                           
##                Accuracy : 0.7784          
##                  95% CI : (0.7654, 0.7911)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.7292          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.75000       0.00000   0.0020450          0.9708
## Specificity                  0.94083       1.00000   1.0000000          0.9761
## Pos Pred Value               0.57670           NaN   1.0000000          0.9348
## Neg Pred Value               0.97223       0.96176   0.8803628          0.9896
## Prevalence                   0.09706       0.03824   0.1198529          0.2605
## Detection Rate               0.07279       0.00000   0.0002451          0.2529
## Detection Prevalence         0.12623       0.00000   0.0002451          0.2706
## Balanced Accuracy            0.84541       0.50000   0.5010225          0.9735
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9567       0.9671      0.8924
## Specificity                0.9129       0.9888      0.9179
## Pos Pred Value             0.6445       0.9378      0.7231
## Neg Pred Value             0.9922       0.9942      0.9726
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1355       0.1441      0.1728
## Detection Prevalence       0.2103       0.1537      0.2390
## Balanced Accuracy          0.9348       0.9779      0.9052
ad_tda_kde_5.50.5_n3_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.7784314      0.7292187      0.7653685      0.7910943      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
ad_tda_kde_5.50.5_n3_rf_cf0_ov_acc<-ad_tda_kde_5.50.5_n3_rf_cf0$overall[1]
ad_tda_kde_5.50.5_n3_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA  0.75000000   0.9408252      0.5766990      0.9722300 0.5766990
## Class: BOMBAY    0.00000000   1.0000000            NaN      0.9617647        NA
## Class: CALI      0.00204499   1.0000000      1.0000000      0.8803628 1.0000000
## Class: DERMASON  0.97083725   0.9761352      0.9347826      0.9895833 0.9347826
## Class: HOROZ     0.95674740   0.9129069      0.6445221      0.9922408 0.6445221
## Class: SEKER     0.96710526   0.9887673      0.9377990      0.9942079 0.9377990
## Class: SIRA      0.89240506   0.9179331      0.7230769      0.9726248 0.7230769
##                     Recall          F1 Prevalence Detection Rate
## Class: BARBUNYA 0.75000000 0.652030735 0.09705882    0.072794118
## Class: BOMBAY   0.00000000          NA 0.03823529    0.000000000
## Class: CALI     0.00204499 0.004081633 0.11985294    0.000245098
## Class: DERMASON 0.97083725 0.952468851 0.26053922    0.252941176
## Class: HOROZ    0.95674740 0.770194986 0.14166667    0.135539216
## Class: SEKER    0.96710526 0.952226721 0.14901961    0.144117647
## Class: SIRA     0.89240506 0.798866856 0.19362745    0.172794118
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.126225490         0.8454126
## Class: BOMBAY            0.000000000         0.5000000
## Class: CALI              0.000245098         0.5010225
## Class: DERMASON          0.270588235         0.9734862
## Class: HOROZ             0.210294118         0.9348272
## Class: SEKER             0.153676471         0.9779363
## Class: SIRA              0.238970588         0.9051691
ad_tda_kde_5.50.5_n3_rf_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n3_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_rf_n3_3_fold<-(db_rf_fit_re-ad_tda_kde_5.50.5_n3_rf_fit0_re)
diff_drybean_tda_kde_5.50.5_rf_n3_3_fold
##      Accuracy
## 1 0.015469785
## 2 0.009173243
## 3 0.015329177
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n3_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n3_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n3_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n3_3_fold_odds.left<-bst_tda_kde_5.50.5_rf.n3_3_fold$probLeft/bst_tda_kde_5.50.5_rf.n3_3_fold$probRight
bst_tda_kde_5.50.5_rf.n3_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n3_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n3_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n3_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.3018667
## 
## $winRight
## [1] 0.6981333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n3_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n3_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n3_3_fold
## $left
## [1] 0.005198295
## 
## $rope
## [1] 0.1447256
## 
## $right
## [1] 0.8500761
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_rf_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf_n3_3_fold))
#bf_tda_kde_5.50.5_rf.n3_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n3_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_rf_n3_3_fold)
## t = 6.4187, df = 2, p-value = 0.02342
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.004392581 0.022255555
## sample estimates:
##  mean of x 
## 0.01332407
### Test set diff
diff_drybean_tda_kde_5.50.5_rf.n3_test<-(db_rf_cf_ov_acc-ad_tda_kde_5.50.5_n3_rf_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_rf.n3_test
##  Accuracy 
## 0.1394608
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n3_test),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n3_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n3_test_odds.left<-bst_tda_kde_5.50.5_rf.n3_test$probLeft/bst_tda_kde_5.50.5_rf.n3_test$probRight
bst_tda_kde_5.50.5_rf.n3_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n3_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n3_test),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n3_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1580333
## 
## $winRight
## [1] 0.8419667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n3_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n3_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n3_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_rf.n3_test))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf.n3_test)) #bf_tda_kde_5.50.5_rf.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n3_test))

##Node4

DryBean_TDA_KDE_5.50.5_n4_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n4.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'),                    
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_KDE_5.50.5_n4_RfFit0
## Random Forest 
## 
## 2024 samples
##   16 predictor
##    4 classes: 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1348, 1351, 1349 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.8196610  0.6997197
##    100  0.8152114  0.6921414
##    150  0.8176879  0.6958127
##    200  0.8171904  0.6949336
##    250  0.8171852  0.6960052
##    300  0.8186667  0.6985042
##    350  0.8147198  0.6919284
##    400  0.8157082  0.6934125
##    450  0.8157067  0.6929295
##    500  0.8166995  0.6948643
##    550  0.8147183  0.6915713
##    600  0.8201460  0.7003374
##    650  0.8181707  0.6973689
##    700  0.8176813  0.6966316
##    750  0.8132354  0.6891959
##    800  0.8166899  0.6956077
##    850  0.8196632  0.6995717
##    900  0.8176871  0.6959447
##    950  0.8191664  0.6990904
##   1000  0.8157060  0.6927469
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 600.
DryBean_TDA_KDE_5.50.5_n4_RfFit0$resample
##    Accuracy     Kappa Resample
## 1 0.8142645 0.6923974    Fold2
## 2 0.8328402 0.7210895    Fold1
## 3 0.8133333 0.6875253    Fold3
ad_tda_kde_5.50.5_n4_rf_fit0_re<-DryBean_TDA_KDE_5.50.5_n4_RfFit0$resample[1]


summary(DryBean_TDA_KDE_5.50.5_n4_RfFit0)
##                 Length Class      Mode     
## call               5   -none-     call     
## type               1   -none-     character
## predicted       2024   factor     numeric  
## err.rate        2500   -none-     numeric  
## confusion         20   -none-     numeric  
## votes           8096   matrix     numeric  
## oob.times       2024   -none-     numeric  
## classes            4   -none-     character
## importance        16   -none-     numeric  
## importanceSD       0   -none-     NULL     
## localImportance    0   -none-     NULL     
## proximity          0   -none-     NULL     
## ntree              1   -none-     numeric  
## mtry               1   -none-     numeric  
## forest            14   -none-     list     
## y               2024   factor     numeric  
## test               0   -none-     NULL     
## inbag              0   -none-     NULL     
## xNames            16   -none-     character
## problemType        1   -none-     character
## tuneValue          1   data.frame list     
## obsLevels          4   -none-     character
## param              1   -none-     list
vip(DryBean_TDA_KDE_5.50.5_n4_RfFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n4_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n4_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n4_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n4_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
ad_tda_kde_5.50.5_n4_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      326     75  342     1015   491    39  337
##   HOROZ           0      0    0        0     3     0    0
##   SEKER          24      5    0        9     0   561    6
##   SIRA           46     76  147       39    84     8  447
## 
## Overall Statistics
##                                          
##                Accuracy : 0.4966         
##                  95% CI : (0.4811, 0.512)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.3462         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9548
## Specificity                  1.00000       1.00000      1.0000          0.4664
## Pos Pred Value                   NaN           NaN         NaN          0.3867
## Neg Pred Value               0.90294       0.96176      0.8801          0.9670
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2488
## Detection Prevalence         0.00000       0.00000      0.0000          0.6434
## Balanced Accuracy            0.50000       0.50000      0.5000          0.7106
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity             0.0051903       0.9227      0.5658
## Specificity             1.0000000       0.9873      0.8784
## Pos Pred Value          1.0000000       0.9273      0.5277
## Neg Pred Value          0.8589649       0.9865      0.8939
## Prevalence              0.1416667       0.1490      0.1936
## Detection Rate          0.0007353       0.1375      0.1096
## Detection Prevalence    0.0007353       0.1483      0.2076
## Balanced Accuracy       0.5025952       0.9550      0.7221
ad_tda_kde_5.50.5_n4_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      326     75  342     1015   491    39  337
##   HOROZ           0      0    0        0     3     0    0
##   SEKER          24      5    0        9     0   561    6
##   SIRA           46     76  147       39    84     8  447
## 
## Overall Statistics
##                                          
##                Accuracy : 0.4966         
##                  95% CI : (0.4811, 0.512)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.3462         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9548
## Specificity                  1.00000       1.00000      1.0000          0.4664
## Pos Pred Value                   NaN           NaN         NaN          0.3867
## Neg Pred Value               0.90294       0.96176      0.8801          0.9670
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2488
## Detection Prevalence         0.00000       0.00000      0.0000          0.6434
## Balanced Accuracy            0.50000       0.50000      0.5000          0.7106
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity             0.0051903       0.9227      0.5658
## Specificity             1.0000000       0.9873      0.8784
## Pos Pred Value          1.0000000       0.9273      0.5277
## Neg Pred Value          0.8589649       0.9865      0.8939
## Prevalence              0.1416667       0.1490      0.1936
## Detection Rate          0.0007353       0.1375      0.1096
## Detection Prevalence    0.0007353       0.1483      0.2076
## Balanced Accuracy       0.5025952       0.9550      0.7221
ad_tda_kde_5.50.5_n4_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   4.965686e-01   3.461723e-01   4.811113e-01   5.120309e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##  5.748350e-227            NaN
ad_tda_kde_5.50.5_n4_rf_cf0_ov_acc<-ad_tda_kde_5.50.5_n4_rf_cf0$overall[1]
ad_tda_kde_5.50.5_n4_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA 0.000000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY   0.000000000   1.0000000            NaN      0.9617647        NA
## Class: CALI     0.000000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON 0.954844779   0.4663573      0.3866667      0.9670103 0.3866667
## Class: HOROZ    0.005190311   1.0000000      1.0000000      0.8589649 1.0000000
## Class: SEKER    0.922697368   0.9873272      0.9272727      0.9864748 0.9272727
## Class: SIRA     0.565822785   0.8784195      0.5277450      0.8939066 0.5277450
##                      Recall         F1 Prevalence Detection Rate
## Class: BARBUNYA 0.000000000         NA 0.09705882   0.0000000000
## Class: BOMBAY   0.000000000         NA 0.03823529   0.0000000000
## Class: CALI     0.000000000         NA 0.11985294   0.0000000000
## Class: DERMASON 0.954844779 0.55043384 0.26053922   0.2487745098
## Class: HOROZ    0.005190311 0.01032702 0.14166667   0.0007352941
## Class: SEKER    0.922697368 0.92497939 0.14901961   0.1375000000
## Class: SIRA     0.565822785 0.54612095 0.19362745   0.1095588235
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA         0.0000000000         0.5000000
## Class: BOMBAY           0.0000000000         0.5000000
## Class: CALI             0.0000000000         0.5000000
## Class: DERMASON         0.6433823529         0.7106010
## Class: HOROZ            0.0007352941         0.5025952
## Class: SEKER            0.1482843137         0.9550123
## Class: SIRA             0.2075980392         0.7221211
ad_tda_kde_5.50.5_n4_rf_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n4_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_rf_n4_3_fold<-(db_rf_fit_re-ad_tda_kde_5.50.5_n4_rf_fit0_re)
diff_drybean_tda_kde_5.50.5_rf_n4_3_fold
##     Accuracy
## 1 0.11082207
## 2 0.09101124
## 3 0.11739715
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n4_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n4_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n4_3_fold_odds.left<-bst_tda_kde_5.50.5_rf.n4_3_fold$probLeft/bst_tda_kde_5.50.5_rf.n4_3_fold$probRight
bst_tda_kde_5.50.5_rf.n4_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n4_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n4_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.008733333
## 
## $winRight
## [1] 0.9912667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n4_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n4_3_fold
## $left
## [1] 0.003065215
## 
## $rope
## [1] 0.001384956
## 
## $right
## [1] 0.9955498
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_rf_n4_3_fold,c(-0.01,0.01)))

### Test set diff
diff_drybean_tda_kde_5.50.5_rf.n4_test<-(db_rf_cf_ov_acc-ad_tda_kde_5.50.5_n4_rf_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_rf.n4_test
##  Accuracy 
## 0.4213235
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n4_test),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 
#BayesFactor
#bf_tda_kde_5.50.5_rf.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf_n4_3_fold))
#bf_tda_kde_5.50.5_rf.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_rf_n4_3_fold)
## t = 13.419, df = 2, p-value = 0.005508
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.07229029 0.14053001
## sample estimates:
## mean of x 
## 0.1064102
bst_tda_kde_5.50.5_rf.n4_test_odds.left<-bst_tda_kde_5.50.5_rf.n4_test$probLeft/bst_tda_kde_5.50.5_rf.n4_test$probRight
bst_tda_kde_5.50.5_rf.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n4_test),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1589667
## 
## $winRight
## [1] 0.8410333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n4_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_rf.n4_test))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf.n4_test)) #bf_tda_kde_5.50.5_rf.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n4_test))

##Node5

DryBean_TDA_KDE_5.50.5_n5_RfFit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n5.vec, 
                 Importance = T,
                 method = 'rf', 
                 trControl = fitControl,
                 tuneGrid = rfGrid, preProc = c('center','scale'),                    
                 metric='Accuracy')
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range

## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
DryBean_TDA_KDE_5.50.5_n5_RfFit0
## Random Forest 
## 
## 989 samples
##  16 predictor
##   4 classes: 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 660, 658, 660 
## Resampling results across tuning parameters:
## 
##   mtry  Accuracy   Kappa    
##     50  0.7259693  0.5224857
##    100  0.7239307  0.5212230
##    150  0.7279651  0.5258962
##    200  0.7148061  0.5022183
##    250  0.7249255  0.5197732
##    300  0.7229114  0.5173059
##    350  0.7269580  0.5245006
##    400  0.7269580  0.5246805
##    450  0.7259510  0.5214657
##    500  0.7269764  0.5240011
##    550  0.7259387  0.5223178
##    600  0.7249378  0.5199934
##    650  0.7229237  0.5190916
##    700  0.7289782  0.5279222
##    750  0.7198719  0.5136756
##    800  0.7259693  0.5215907
##    850  0.7239430  0.5186947
##    900  0.7209035  0.5142262
##    950  0.7198781  0.5148667
##   1000  0.7279589  0.5266978
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 700.
DryBean_TDA_KDE_5.50.5_n5_RfFit0$resample
##    Accuracy     Kappa Resample
## 1 0.7173252 0.5220190    Fold1
## 2 0.7203647 0.4970255    Fold3
## 3 0.7492447 0.5647221    Fold2
ad_tda_kde_5.50.5_n5_rf_fit0_re<-DryBean_TDA_KDE_5.50.5_n5_RfFit0$resample[1]


summary(DryBean_TDA_KDE_5.50.5_n5_RfFit0)
##                 Length Class      Mode     
## call               5   -none-     call     
## type               1   -none-     character
## predicted        989   factor     numeric  
## err.rate        2500   -none-     numeric  
## confusion         20   -none-     numeric  
## votes           3956   matrix     numeric  
## oob.times        989   -none-     numeric  
## classes            4   -none-     character
## importance        16   -none-     numeric  
## importanceSD       0   -none-     NULL     
## localImportance    0   -none-     NULL     
## proximity          0   -none-     NULL     
## ntree              1   -none-     numeric  
## mtry               1   -none-     numeric  
## forest            14   -none-     list     
## y                989   factor     numeric  
## test               0   -none-     NULL     
## inbag              0   -none-     NULL     
## xNames            16   -none-     character
## problemType        1   -none-     character
## tuneValue          1   data.frame list     
## obsLevels          4   -none-     character
## param              1   -none-     list
vip(DryBean_TDA_KDE_5.50.5_n5_RfFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n5_RfFit TDA-Assited RF")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n5_RfFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n5_RfFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n5_rf_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
ad_tda_kde_5.50.5_n5_rf_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      111      7   16      920    14    35  136
##   HOROZ           0      0    0        0     1     0    0
##   SEKER          12      4    0       18     0   562    9
##   SIRA          273    145  473      125   563    11  645
## 
## Overall Statistics
##                                          
##                Accuracy : 0.5216         
##                  95% CI : (0.5061, 0.537)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.3964         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.8655
## Specificity                  1.00000       1.00000      1.0000          0.8943
## Pos Pred Value                   NaN           NaN         NaN          0.7425
## Neg Pred Value               0.90294       0.96176      0.8801          0.9497
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2255
## Detection Prevalence         0.00000       0.00000      0.0000          0.3037
## Balanced Accuracy            0.50000       0.50000      0.5000          0.8799
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity             0.0017301       0.9243      0.8165
## Specificity             1.0000000       0.9876      0.5167
## Pos Pred Value          1.0000000       0.9289      0.2886
## Neg Pred Value          0.8585438       0.9868      0.9214
## Prevalence              0.1416667       0.1490      0.1936
## Detection Rate          0.0002451       0.1377      0.1581
## Detection Prevalence    0.0002451       0.1483      0.5478
## Balanced Accuracy       0.5008651       0.9560      0.6666
ad_tda_kde_5.50.5_n5_rf_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      111      7   16      920    14    35  136
##   HOROZ           0      0    0        0     1     0    0
##   SEKER          12      4    0       18     0   562    9
##   SIRA          273    145  473      125   563    11  645
## 
## Overall Statistics
##                                          
##                Accuracy : 0.5216         
##                  95% CI : (0.5061, 0.537)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.3964         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.8655
## Specificity                  1.00000       1.00000      1.0000          0.8943
## Pos Pred Value                   NaN           NaN         NaN          0.7425
## Neg Pred Value               0.90294       0.96176      0.8801          0.9497
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2255
## Detection Prevalence         0.00000       0.00000      0.0000          0.3037
## Balanced Accuracy            0.50000       0.50000      0.5000          0.8799
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity             0.0017301       0.9243      0.8165
## Specificity             1.0000000       0.9876      0.5167
## Pos Pred Value          1.0000000       0.9289      0.2886
## Neg Pred Value          0.8585438       0.9868      0.9214
## Prevalence              0.1416667       0.1490      0.1936
## Detection Rate          0.0002451       0.1377      0.1581
## Detection Prevalence    0.0002451       0.1483      0.5478
## Balanced Accuracy       0.5008651       0.9560      0.6666
ad_tda_kde_5.50.5_n5_rf_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   5.215686e-01   3.964385e-01   5.061073e-01   5.369990e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##  8.315958e-275            NaN
ad_tda_kde_5.50.5_n5_rf_cf0_ov_acc<-ad_tda_kde_5.50.5_n5_rf_cf0$overall[1]
ad_tda_kde_5.50.5_n5_rf_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA 0.000000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY   0.000000000   1.0000000            NaN      0.9617647        NA
## Class: CALI     0.000000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON 0.865475071   0.8942658      0.7425343      0.9496656 0.7425343
## Class: HOROZ    0.001730104   1.0000000      1.0000000      0.8585438 1.0000000
## Class: SEKER    0.924342105   0.9876152      0.9289256      0.9867626 0.9289256
## Class: SIRA     0.816455696   0.5167173      0.2885906      0.9214092 0.2885906
##                      Recall          F1 Prevalence Detection Rate
## Class: BARBUNYA 0.000000000          NA 0.09705882    0.000000000
## Class: BOMBAY   0.000000000          NA 0.03823529    0.000000000
## Class: CALI     0.000000000          NA 0.11985294    0.000000000
## Class: DERMASON 0.865475071 0.799304952 0.26053922    0.225490196
## Class: HOROZ    0.001730104 0.003454231 0.14166667    0.000245098
## Class: SEKER    0.924342105 0.926628195 0.14901961    0.137745098
## Class: SIRA     0.816455696 0.426446281 0.19362745    0.158088235
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.000000000         0.5000000
## Class: BOMBAY            0.000000000         0.5000000
## Class: CALI              0.000000000         0.5000000
## Class: DERMASON          0.303676471         0.8798704
## Class: HOROZ             0.000245098         0.5008651
## Class: SEKER             0.148284314         0.9559787
## Class: SIRA              0.547794118         0.6665865
ad_tda_kde_5.50.5_n5_rf_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n5_rf_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_rf_n5_3_fold<-(db_rf_fit_re-ad_tda_kde_5.50.5_n5_rf_fit0_re)
diff_drybean_tda_kde_5.50.5_rf_n5_3_fold
##    Accuracy
## 1 0.2077613
## 2 0.2034867
## 3 0.1814858
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n5_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n5_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n5_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n5_3_fold_odds.left<-bst_tda_kde_5.50.5_rf.n5_3_fold$probLeft/bst_tda_kde_5.50.5_rf.n5_3_fold$probRight
bst_tda_kde_5.50.5_rf.n5_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n5_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n5_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n5_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.008866667
## 
## $winRight
## [1] 0.9911333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n5_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n5_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n5_3_fold
## $left
## [1] 0.001022067
## 
## $rope
## [1] 0.0002287087
## 
## $right
## [1] 0.9987492
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_rf_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf_n5_3_fold))
#bf_tda_kde_5.50.5_rf.n5_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf_n5_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_rf_n5_3_fold)
## t = 24.272, df = 2, p-value = 0.001693
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.1625537 0.2326022
## sample estimates:
## mean of x 
## 0.1975779
### Test set diff
diff_drybean_tda_kde_5.50.5_rf.n5_test<-(db_rf_cf_ov_acc-ad_tda_kde_5.50.5_n5_rf_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_rf.n5_test
##  Accuracy 
## 0.3963235
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_rf.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n5_test),-0.01,0.01)
bst_tda_kde_5.50.5_rf.n5_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_rf.n5_test_odds.left<-bst_tda_kde_5.50.5_rf.n5_test$probLeft/bst_tda_kde_5.50.5_rf.n5_test$probRight
bst_tda_kde_5.50.5_rf.n5_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_rf.n5_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n5_test),-0.01,0.01)
bsr_tda_kde_5.50.5_rf.n5_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1595667
## 
## $winRight
## [1] 0.8404333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_rf.n5_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n5_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_rf.n5_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_rf.n5_test))

#BayesFactor
#bf_tda_kde_5.50.5_rf.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_rf.n5_test)) #bf_tda_kde_5.50.5_rf.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_rf.n5_test))

##Non-TDA-Assisted

svmGrid<-expand.grid(sigma = c(0.1, 1, 10), C = (1:5*0.25))

#Support Vector Machine-Radial Basis 
dryBeanSvmFit <- train(as.factor(Class) ~ ., data = Dry_Bean_DatasetTrain, 
                   Importance = T,
                   method = 'svmRadial', 
                           trControl = fitControl,
           tuneGrid = svmGrid, preProc = c('center','scale'),
                           metric='Accuracy')

dryBeanSvmFit
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 9531 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6353, 6355, 6354 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa    
##    0.1   0.25  0.9268711  0.9115196
##    0.1   0.50  0.9275003  0.9122810
##    0.1   0.75  0.9290742  0.9141951
##    0.1   1.00  0.9291794  0.9143259
##    0.1   1.25  0.9292842  0.9144525
##    1.0   0.25  0.8977036  0.8761324
##    1.0   0.50  0.9112381  0.8925862
##    1.0   0.75  0.9159600  0.8983123
##    1.0   1.00  0.9178480  0.9006105
##    1.0   1.25  0.9181629  0.9009961
##   10.0   0.25  0.3487567  0.1295442
##   10.0   0.50  0.4499008  0.2742981
##   10.0   0.75  0.5237639  0.3786787
##   10.0   1.00  0.6038176  0.4900585
##   10.0   1.25  0.6312022  0.5277824
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
dryBeanSvmFit$resample
##    Accuracy     Kappa Resample
## 1 0.9238515 0.9078548    Fold1
## 2 0.9320113 0.9177619    Fold3
## 3 0.9319899 0.9177409    Fold2
db_svm_fit_re<-dryBeanSvmFit$resample[1]

summary(dryBeanSvmFit)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(dryBeanSvmFit, 25) + ggtitle("non-TDA-Assited Svm")

# Predict outcome using model from training data based on testing data
predictions <- predict(dryBeanSvmFit, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_svm_cf<-confusionMatrix(data=predictions, as.factor(Dry_Bean_DatasetTest$Class))
db_svm_cf
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      366      0   12        0     0     4    3
##   BOMBAY          1    156    0        0     0     0    0
##   CALI           15      0  466        0     8     0    1
##   DERMASON        0      0    0      971     6    10   80
##   HOROZ           3      0    4        1   556     0   12
##   SEKER           2      0    1       20     0   576    3
##   SIRA            9      0    6       71     8    18  691
## 
## Overall Statistics
##                                           
##                Accuracy : 0.927           
##                  95% CI : (0.9185, 0.9348)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.9117          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.92424       1.00000      0.9530          0.9135
## Specificity                  0.99484       0.99975      0.9933          0.9682
## Pos Pred Value               0.95065       0.99363      0.9510          0.9100
## Neg Pred Value               0.99188       1.00000      0.9936          0.9695
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08971       0.03824      0.1142          0.2380
## Detection Prevalence         0.09436       0.03848      0.1201          0.2615
## Balanced Accuracy            0.95954       0.99987      0.9731          0.9408
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9619       0.9474      0.8747
## Specificity                0.9943       0.9925      0.9660
## Pos Pred Value             0.9653       0.9568      0.8605
## Neg Pred Value             0.9937       0.9908      0.9698
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1363       0.1412      0.1694
## Detection Prevalence       0.1412       0.1475      0.1968
## Balanced Accuracy          0.9781       0.9699      0.9203
db_svm_cf$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.9269608      0.9116543      0.9185443      0.9347595      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_svm_cf_ov_acc<-db_svm_cf$overall[1]
db_svm_cf$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9242424   0.9948426      0.9506494      0.9918809 0.9506494
## Class: BOMBAY     1.0000000   0.9997452      0.9936306      1.0000000 0.9936306
## Class: CALI       0.9529652   0.9933166      0.9510204      0.9935933 0.9510204
## Class: DERMASON   0.9134525   0.9681803      0.9100281      0.9694656 0.9100281
## Class: HOROZ      0.9619377   0.9942890      0.9652778      0.9937215 0.9652778
## Class: SEKER      0.9473684   0.9925115      0.9568106      0.9907993 0.9568106
## Class: SIRA       0.8746835   0.9659574      0.8605230      0.9697894 0.8605230
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9242424 0.9372599 0.09705882     0.08970588
## Class: BOMBAY   1.0000000 0.9968051 0.03823529     0.03823529
## Class: CALI     0.9529652 0.9519918 0.11985294     0.11421569
## Class: DERMASON 0.9134525 0.9117371 0.26053922     0.23799020
## Class: HOROZ    0.9619377 0.9636049 0.14166667     0.13627451
## Class: SEKER    0.9473684 0.9520661 0.14901961     0.14117647
## Class: SIRA     0.8746835 0.8675455 0.19362745     0.16936275
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09436275         0.9595425
## Class: BOMBAY             0.03848039         0.9998726
## Class: CALI               0.12009804         0.9731409
## Class: DERMASON           0.26151961         0.9408164
## Class: HOROZ              0.14117647         0.9781133
## Class: SEKER              0.14754902         0.9699400
## Class: SIRA               0.19681373         0.9203205
db_svm_cf_pr_rec_f1<-db_svm_cf$byClass[5:7]

##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node1

DryBean_TDA_PC_5.50.5_n1_SvmFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n1.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_PC_5.50.5_n1_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 7839 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5225, 5227, 5226 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa       
##    0.1   0.25  0.9113419  0.8627017920
##    0.1   0.50  0.9135106  0.8661255376
##    0.1   0.75  0.9142758  0.8673248524
##    0.1   1.00  0.9146582  0.8679626002
##    0.1   1.25  0.9154233  0.8691065270
##    1.0   0.25  0.8748554  0.8028396839
##    1.0   0.50  0.8877401  0.8241868748
##    1.0   0.75  0.8957772  0.8372588298
##    1.0   1.00  0.8974359  0.8401116413
##    1.0   1.25  0.8987114  0.8422065390
##   10.0   0.25  0.4524811  0.0008332259
##   10.0   0.50  0.4730200  0.0452479889
##   10.0   0.75  0.5098869  0.1233115931
##   10.0   1.00  0.5533870  0.2136329066
##   10.0   1.25  0.5865545  0.2813179245
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
DryBean_TDA_PC_5.50.5_n1_SvmFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9123948 0.8643032    Fold1
## 2 0.9184845 0.8740524    Fold3
## 3 0.9153905 0.8689640    Fold2
db_tda_pc_5.50.5_n1_svm_fit_re<-DryBean_TDA_PC_5.50.5_n1_SvmFit0 $resample[1]

summary(DryBean_TDA_PC_5.50.5_n1_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_PC_5.50.5_n1_SvmFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n1_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_PC_5.50.5_n1_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n1_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n1_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n1_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA       53      0    0        0     0     2    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      277    156  455      996   533     8  101
##   HOROZ           0      0    0        0     0     0    0
##   SEKER          54      0   29       12     0   583    6
##   SIRA           12      0    5       55    45    15  683
## 
## Overall Statistics
##                                          
##                Accuracy : 0.5674         
##                  95% CI : (0.552, 0.5827)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.4409         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.13384       0.00000      0.0000          0.9370
## Specificity                  0.99946       1.00000      1.0000          0.4929
## Pos Pred Value               0.96364           NaN         NaN          0.3943
## Neg Pred Value               0.91478       0.96176      0.8801          0.9569
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.01299       0.00000      0.0000          0.2441
## Detection Prevalence         0.01348       0.00000      0.0000          0.6191
## Balanced Accuracy            0.56665       0.50000      0.5000          0.7149
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000       0.9589      0.8646
## Specificity                1.0000       0.9709      0.9599
## Pos Pred Value                NaN       0.8523      0.8380
## Neg Pred Value             0.8583       0.9926      0.9672
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.0000       0.1429      0.1674
## Detection Prevalence       0.0000       0.1676      0.1998
## Balanced Accuracy          0.5000       0.9649      0.9122
db_tda_pc_5.50.5_n1_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA       53      0    0        0     0     2    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      277    156  455      996   533     8  101
##   HOROZ           0      0    0        0     0     0    0
##   SEKER          54      0   29       12     0   583    6
##   SIRA           12      0    5       55    45    15  683
## 
## Overall Statistics
##                                          
##                Accuracy : 0.5674         
##                  95% CI : (0.552, 0.5827)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.4409         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.13384       0.00000      0.0000          0.9370
## Specificity                  0.99946       1.00000      1.0000          0.4929
## Pos Pred Value               0.96364           NaN         NaN          0.3943
## Neg Pred Value               0.91478       0.96176      0.8801          0.9569
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.01299       0.00000      0.0000          0.2441
## Detection Prevalence         0.01348       0.00000      0.0000          0.6191
## Balanced Accuracy            0.56665       0.50000      0.5000          0.7149
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000       0.9589      0.8646
## Specificity                1.0000       0.9709      0.9599
## Pos Pred Value                NaN       0.8523      0.8380
## Neg Pred Value             0.8583       0.9926      0.9672
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.0000       0.1429      0.1674
## Detection Prevalence       0.0000       0.1676      0.1998
## Balanced Accuracy          0.5000       0.9649      0.9122
db_tda_pc_5.50.5_n1_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.5674020      0.4408902      0.5520335      0.5826738      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n1_db_svm_cf0_ov_acc<-db_tda_pc_5.50.5_n1_db_svm_cf0$overall[1]
db_tda_pc_5.50.5_n1_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.1338384   0.9994571      0.9636364      0.9147826 0.9636364
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.0000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON   0.9369708   0.4928737      0.3942993      0.9568855 0.3942993
## Class: HOROZ      0.0000000   1.0000000            NaN      0.8583333        NA
## Class: SEKER      0.9588816   0.9709101      0.8523392      0.9926384 0.8523392
## Class: SIRA       0.8645570   0.9598784      0.8380368      0.9672282 0.8380368
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.1338384 0.2350333 0.09705882      0.0129902
## Class: BOMBAY   0.0000000        NA 0.03823529      0.0000000
## Class: CALI     0.0000000        NA 0.11985294      0.0000000
## Class: DERMASON 0.9369708 0.5550293 0.26053922      0.2441176
## Class: HOROZ    0.0000000        NA 0.14166667      0.0000000
## Class: SEKER    0.9588816 0.9024768 0.14901961      0.1428922
## Class: SIRA     0.8645570 0.8510903 0.19362745      0.1674020
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.01348039         0.5666477
## Class: BOMBAY             0.00000000         0.5000000
## Class: CALI               0.00000000         0.5000000
## Class: DERMASON           0.61911765         0.7149223
## Class: HOROZ              0.00000000         0.5000000
## Class: SEKER              0.16764706         0.9648959
## Class: SIRA               0.19975490         0.9122177
db_tda_pc_5.50.5_n1_db_svm_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n1_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_svm_n1_3_fold<-(db_svm_fit_re - db_tda_pc_5.50.5_n1_svm_fit_re)
diff_drybean_tda_pca_5.50.5_svm_n1_3_fold
##     Accuracy
## 1 0.01145668
## 2 0.01352683
## 3 0.01659942
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n1_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n1_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n1_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n1_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n1_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n1_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n1_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n1_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1681667
## 
## $winRight
## [1] 0.8318333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n1_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n1_3_fold
## $left
## [1] 0.002593085
## 
## $rope
## [1] 0.07471691
## 
## $right
## [1] 0.92269
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_svm_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm_n1_3_fold))
#bf_tda_pca_5.50.5_rf.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_svm_n1_3_fold)
## t = 9.2781, df = 2, p-value = 0.01142
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.007433021 0.020288934
## sample estimates:
##  mean of x 
## 0.01386098
### Test set diff
diff_drybean_tda_pca_5.50.5_svm.n1_test<-(db_svm_cf_ov_acc - db_tda_pc_5.50.5_n1_db_svm_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_svm.n1_test
##  Accuracy 
## 0.3595588
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n1_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n1_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n1_test$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n1_test$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n1_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n1_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1627333
## 
## $winRight
## [1] 0.8372667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n1_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_svm.n1_test)))

#BayesFactor
#bf_tda_pca_5.50.5_svm.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm.n1_test)) #bf_tda_pca_5.50.5_svm.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n1_test))

##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node2

DryBean_TDA_PC_5.50.5_n2_SvmFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n2.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_PC_5.50.5_n2_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 9515 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6344, 6343, 6343 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa     
##    0.1   0.25  0.6442739  0.59036543
##    0.1   0.50  0.6446941  0.59073778
##    0.1   0.75  0.6440636  0.58993080
##    0.1   1.00  0.6449044  0.59090851
##    0.1   1.25  0.6450095  0.59099313
##    1.0   0.25  0.6322921  0.57402305
##    1.0   0.50  0.6367059  0.57984296
##    1.0   0.75  0.6394384  0.58353174
##    1.0   1.00  0.6410151  0.58565467
##    1.0   1.25  0.6408050  0.58537991
##   10.0   0.25  0.2299623  0.03489697
##   10.0   0.50  0.2921832  0.12100639
##   10.0   0.75  0.3241348  0.16619282
##   10.0   1.00  0.3518817  0.20539323
##   10.0   1.25  0.3689081  0.22948435
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
DryBean_TDA_PC_5.50.5_n2_SvmFit0$resample
##    Accuracy       Kappa Resample
## 1 0.9101230 0.886555766    Fold1
## 2 0.1210593 0.007868807    Fold3
## 3 0.9038462 0.878554819    Fold2
db_tda_pc_5.50.5_n2_svm_fit_re<-DryBean_TDA_PC_5.50.5_n2_SvmFit0 $resample[1]

summary(DryBean_TDA_PC_5.50.5_n2_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_PC_5.50.5_n2_SvmFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n2_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_PC_5.50.5_n2_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n2_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n2_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
db_tda_pc_5.50.5_n2_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      370      1   22        0     3     8    3
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           12      0  451        0     5     0    1
##   DERMASON        0      0    0      986     6    17   84
##   HOROZ           3    155   10        1   556     2    8
##   SEKER           3      0    1       16     0   564    3
##   SIRA            8      0    5       60     8    17  691
## 
## Overall Statistics
##                                           
##                Accuracy : 0.8868          
##                  95% CI : (0.8766, 0.8963)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8623          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.93434       0.00000      0.9223          0.9276
## Specificity                  0.98996       1.00000      0.9950          0.9645
## Pos Pred Value               0.90909           NaN      0.9616          0.9021
## Neg Pred Value               0.99292       0.96176      0.9895          0.9742
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.09069       0.00000      0.1105          0.2417
## Detection Prevalence         0.09975       0.00000      0.1150          0.2679
## Balanced Accuracy            0.96215       0.50000      0.9586          0.9460
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9619       0.9276      0.8747
## Specificity                0.9489       0.9934      0.9702
## Pos Pred Value             0.7565       0.9608      0.8758
## Neg Pred Value             0.9934       0.9874      0.9699
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1363       0.1382      0.1694
## Detection Prevalence       0.1801       0.1439      0.1934
## Balanced Accuracy          0.9554       0.9605      0.9224
db_tda_pc_5.50.5_n2_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      370      1   22        0     3     8    3
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           12      0  451        0     5     0    1
##   DERMASON        0      0    0      986     6    17   84
##   HOROZ           3    155   10        1   556     2    8
##   SEKER           3      0    1       16     0   564    3
##   SIRA            8      0    5       60     8    17  691
## 
## Overall Statistics
##                                           
##                Accuracy : 0.8868          
##                  95% CI : (0.8766, 0.8963)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8623          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.93434       0.00000      0.9223          0.9276
## Specificity                  0.98996       1.00000      0.9950          0.9645
## Pos Pred Value               0.90909           NaN      0.9616          0.9021
## Neg Pred Value               0.99292       0.96176      0.9895          0.9742
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.09069       0.00000      0.1105          0.2417
## Detection Prevalence         0.09975       0.00000      0.1150          0.2679
## Balanced Accuracy            0.96215       0.50000      0.9586          0.9460
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9619       0.9276      0.8747
## Specificity                0.9489       0.9934      0.9702
## Pos Pred Value             0.7565       0.9608      0.8758
## Neg Pred Value             0.9934       0.9874      0.9699
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1363       0.1382      0.1694
## Detection Prevalence       0.1801       0.1439      0.1934
## Balanced Accuracy          0.9554       0.9605      0.9224
db_tda_pc_5.50.5_n2_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.8867647      0.8623010      0.8766407      0.8963310      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n2_db_svm_cf0_ov_acc<-db_tda_pc_5.50.5_n2_db_svm_cf0$overall[1]
db_tda_pc_5.50.5_n2_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9343434   0.9899566      0.9090909      0.9929213 0.9090909
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.9222904   0.9949875      0.9616205      0.9894766 0.9616205
## Class: DERMASON   0.9275635   0.9645343      0.9021043      0.9742216 0.9021043
## Class: HOROZ      0.9619377   0.9488864      0.7564626      0.9934230 0.7564626
## Class: SEKER      0.9276316   0.9933756      0.9608177      0.9874034 0.9608177
## Class: SIRA       0.8746835   0.9702128      0.8757921      0.9699180 0.8757921
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9343434 0.9215442 0.09705882     0.09068627
## Class: BOMBAY   0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.9222904 0.9415449 0.11985294     0.11053922
## Class: DERMASON 0.9275635 0.9146568 0.26053922     0.24166667
## Class: HOROZ    0.9619377 0.8469155 0.14166667     0.13627451
## Class: SEKER    0.9276316 0.9439331 0.14901961     0.13823529
## Class: SIRA     0.8746835 0.8752375 0.19362745     0.16936275
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA            0.0997549         0.9621500
## Class: BOMBAY              0.0000000         0.5000000
## Class: CALI                0.1149510         0.9586389
## Class: DERMASON            0.2678922         0.9460489
## Class: HOROZ               0.1801471         0.9554120
## Class: SEKER               0.1438725         0.9605036
## Class: SIRA                0.1933824         0.9224482
db_tda_pc_5.50.5_n2_db_svm_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n2_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_svm_n2_3_fold<-(db_svm_fit_re - db_tda_pc_5.50.5_n2_svm_fit_re)
diff_drybean_tda_pca_5.50.5_svm_n2_3_fold
##     Accuracy
## 1 0.01372849
## 2 0.81095206
## 3 0.02814377
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n2_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n2_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n2_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n2_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n2_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n2_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n2_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n2_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n2_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n2_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.03776667
## 
## $winRight
## [1] 0.9622333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n2_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n2_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n2_3_fold
## $left
## [1] 0.2176536
## 
## $rope
## [1] 0.01350065
## 
## $right
## [1] 0.7688458
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_svm_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm_n2_3_fold))
#bf_tda_pca_5.50.5_rf.n2_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n2_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_svm_n2_3_fold)
## t = 1.0794, df = 2, p-value = 0.3933
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.8489214  1.4174710
## sample estimates:
## mean of x 
## 0.2842748
### Test set diff
diff_drybean_tda_pca_5.50.5_svm.n2_test<-(db_svm_cf_ov_acc - db_tda_pc_5.50.5_n2_db_svm_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_svm.n2_test
##   Accuracy 
## 0.04019608
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n2_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n2_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n2_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n2_test$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n2_test$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n2_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n2_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1584
## 
## $winRight
## [1] 0.8416
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n2_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n2_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n2_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_svm.n2_test)))

#BayesFactor
#bf_tda_pca_5.50.5_svm.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm.n2_test)) #bf_tda_pca_5.50.5_svm.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n2_test))

##Node3

DryBean_TDA_PC_5.50.5_n3_SvmFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n3.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_PC_5.50.5_n3_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 5355 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 3569, 3572, 3569 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa      
##    0.1   0.25  0.9443508  0.922342078
##    0.1   0.50  0.9460328  0.924707408
##    0.1   0.75  0.9462194  0.924997413
##    0.1   1.00  0.9467790  0.925787231
##    0.1   1.25  0.9473395  0.926572092
##    1.0   0.25  0.9116683  0.875162547
##    1.0   0.50  0.9254905  0.895305238
##    1.0   0.75  0.9299716  0.901796732
##    1.0   1.00  0.9329603  0.906046503
##    1.0   1.25  0.9329600  0.906138271
##   10.0   0.25  0.3555556  0.000000000
##   10.0   0.50  0.3598510  0.007010944
##   10.0   0.75  0.4089632  0.087523580
##   10.0   1.00  0.5069990  0.246940012
##   10.0   1.25  0.5353810  0.293893563
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
DryBean_TDA_PC_5.50.5_n3_SvmFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9524076 0.9337115    Fold1
## 2 0.9412094 0.9179588    Fold3
## 3 0.9484016 0.9280459    Fold2
db_tda_pc_5.50.5_n3_svm_fit_re<-DryBean_TDA_PC_5.50.5_n3_SvmFit0 $resample[1]

summary(DryBean_TDA_PC_5.50.5_n3_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_PC_5.50.5_n3_SvmFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n3_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_PC_5.50.5_n3_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n3_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n3_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
db_tda_pc_5.50.5_n3_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      372      0   14       44     0    59  143
##   BOMBAY          0     83    0        0     0     0    0
##   CALI           17      0  467        0     8     0    1
##   DERMASON        0      0    0        1     0     0    0
##   HOROZ           4     73    4      987   561   546  111
##   SEKER           0      0    0        0     0     0    0
##   SIRA            3      0    4       31     9     3  535
## 
## Overall Statistics
##                                           
##                Accuracy : 0.4949          
##                  95% CI : (0.4794, 0.5103)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.4143          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.93939       0.53205      0.9550       0.0009407
## Specificity                  0.92942       1.00000      0.9928       1.0000000
## Pos Pred Value               0.58861       1.00000      0.9473       1.0000000
## Neg Pred Value               0.99304       0.98174      0.9939       0.7396421
## Prevalence                   0.09706       0.03824      0.1199       0.2605392
## Detection Rate               0.09118       0.02034      0.1145       0.0002451
## Detection Prevalence         0.15490       0.02034      0.1208       0.0002451
## Balanced Accuracy            0.93441       0.76603      0.9739       0.5004704
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9706        0.000      0.6772
## Specificity                0.5074        1.000      0.9848
## Pos Pred Value             0.2454          NaN      0.9145
## Neg Pred Value             0.9905        0.851      0.9270
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1375        0.000      0.1311
## Detection Prevalence       0.5603        0.000      0.1434
## Balanced Accuracy          0.7390        0.500      0.8310
db_tda_pc_5.50.5_n3_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      372      0   14       44     0    59  143
##   BOMBAY          0     83    0        0     0     0    0
##   CALI           17      0  467        0     8     0    1
##   DERMASON        0      0    0        1     0     0    0
##   HOROZ           4     73    4      987   561   546  111
##   SEKER           0      0    0        0     0     0    0
##   SIRA            3      0    4       31     9     3  535
## 
## Overall Statistics
##                                           
##                Accuracy : 0.4949          
##                  95% CI : (0.4794, 0.5103)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.4143          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.93939       0.53205      0.9550       0.0009407
## Specificity                  0.92942       1.00000      0.9928       1.0000000
## Pos Pred Value               0.58861       1.00000      0.9473       1.0000000
## Neg Pred Value               0.99304       0.98174      0.9939       0.7396421
## Prevalence                   0.09706       0.03824      0.1199       0.2605392
## Detection Rate               0.09118       0.02034      0.1145       0.0002451
## Detection Prevalence         0.15490       0.02034      0.1208       0.0002451
## Balanced Accuracy            0.93441       0.76603      0.9739       0.5004704
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9706        0.000      0.6772
## Specificity                0.5074        1.000      0.9848
## Pos Pred Value             0.2454          NaN      0.9145
## Neg Pred Value             0.9905        0.851      0.9270
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1375        0.000      0.1311
## Detection Prevalence       0.5603        0.000      0.1434
## Balanced Accuracy          0.7390        0.500      0.8310
db_tda_pc_5.50.5_n3_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   4.948529e-01   4.143248e-01   4.793973e-01   5.103159e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##  7.591924e-224            NaN
db_tda_pc_5.50.5_n3_db_svm_cf0_ov_acc<-db_tda_pc_5.50.5_n3_db_svm_cf0$overall[1]
db_tda_pc_5.50.5_n3_db_svm_cf0$byClass
##                  Sensitivity Specificity Pos Pred Value Neg Pred Value
## Class: BARBUNYA 0.9393939394   0.9294245      0.5886076      0.9930394
## Class: BOMBAY   0.5320512821   1.0000000      1.0000000      0.9817363
## Class: CALI     0.9550102249   0.9927597      0.9472617      0.9938667
## Class: DERMASON 0.0009407338   1.0000000      1.0000000      0.7396421
## Class: HOROZ    0.9705882353   0.5074243      0.2454068      0.9905240
## Class: SEKER    0.0000000000   1.0000000            NaN      0.8509804
## Class: SIRA     0.6772151899   0.9848024      0.9145299      0.9270386
##                 Precision       Recall          F1 Prevalence Detection Rate
## Class: BARBUNYA 0.5886076 0.9393939394 0.723735409 0.09705882    0.091176471
## Class: BOMBAY   1.0000000 0.5320512821 0.694560669 0.03823529    0.020343137
## Class: CALI     0.9472617 0.9550102249 0.951120163 0.11985294    0.114460784
## Class: DERMASON 1.0000000 0.0009407338 0.001879699 0.26053922    0.000245098
## Class: HOROZ    0.2454068 0.9705882353 0.391759777 0.14166667    0.137500000
## Class: SEKER           NA 0.0000000000          NA 0.14901961    0.000000000
## Class: SIRA     0.9145299 0.6772151899 0.778181818 0.19362745    0.131127451
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.154901961         0.9344092
## Class: BOMBAY            0.020343137         0.7660256
## Class: CALI              0.120833333         0.9738850
## Class: DERMASON          0.000245098         0.5004704
## Class: HOROZ             0.560294118         0.7390063
## Class: SEKER             0.000000000         0.5000000
## Class: SIRA              0.143382353         0.8310088
db_tda_pc_5.50.5_n3_db_svm_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n3_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_svm_n3_3_fold<-(db_svm_fit_re - db_tda_pc_5.50.5_n3_svm_fit_re)
diff_drybean_tda_pca_5.50.5_svm_n3_3_fold
##       Accuracy
## 1 -0.028556136
## 2 -0.009198075
## 3 -0.016411646
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n3_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n3_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n3_3_fold
## $probLeft
## [1] 0.5
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n3_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n3_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n3_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n3_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n3_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n3_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n3_3_fold
## $winLeft
## [1] 0.7847
## 
## $winRope
## [1] 0.2153
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n3_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n3_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n3_3_fold
## $left
## [1] 0.8288962
## 
## $rope
## [1] 0.1460922
## 
## $right
## [1] 0.02501158
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_svm_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm_n3_3_fold))
#bf_tda_pca_5.50.5_rf.n3_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n3_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_svm_n3_3_fold)
## t = -3.1966, df = 2, p-value = 0.0855
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.042357948  0.006247377
## sample estimates:
##   mean of x 
## -0.01805529
### Test set diff
diff_drybean_tda_pca_5.50.5_svm.n3_test<-(db_svm_cf_ov_acc - db_tda_pc_5.50.5_n3_db_svm_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_svm.n3_test
##  Accuracy 
## 0.4321078
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n3_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n3_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n3_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n3_test$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n3_test$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n3_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n3_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n3_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n3_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1607
## 
## $winRight
## [1] 0.8393
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n3_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n3_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n3_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_svm.n3_test)))

#BayesFactor
#bf_tda_pca_5.50.5_svm.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm.n3_test)) #bf_tda_pca_5.50.5_svm.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n3_test))


##Node4

DryBean_TDA_PC_5.50.5_n4_SvmFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n4.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_PC_5.50.5_n4_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 1590 samples
##   16 predictor
##    4 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'HOROZ' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1060, 1059, 1061 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa     
##    0.1   0.25  0.9779743  0.96780604
##    0.1   0.50  0.9792310  0.96966610
##    0.1   0.75  0.9792298  0.96967461
##    0.1   1.00  0.9792298  0.96967461
##    0.1   1.25  0.9811190  0.97245730
##    1.0   0.25  0.9383634  0.90870262
##    1.0   0.50  0.9559688  0.93561096
##    1.0   0.75  0.9609931  0.94301438
##    1.0   1.00  0.9653957  0.94943421
##    1.0   1.25  0.9653980  0.94945911
##   10.0   0.25  0.4201256  0.00000000
##   10.0   0.50  0.4213846  0.00253303
##   10.0   0.75  0.4446599  0.04877826
##   10.0   1.00  0.4905684  0.13663762
##   10.0   1.25  0.5257862  0.20388946
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
DryBean_TDA_PC_5.50.5_n4_SvmFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9867925 0.9808476    Fold1
## 2 0.9678639 0.9530419    Fold3
## 3 0.9887006 0.9834825    Fold2
db_tda_pc_5.50.5_n4_svm_fit_re<-DryBean_TDA_PC_5.50.5_n4_SvmFit0 $resample[1]

summary(DryBean_TDA_PC_5.50.5_n4_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_PC_5.50.5_n4_SvmFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n4_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_PC_5.50.5_n4_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n4_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n4_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n4_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      252      0    6        0     0     0    1
##   BOMBAY          5    156    0        0     0     0    0
##   CALI           31      0  462        0    22     0   10
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ         108      0   21     1063   556   608  779
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3495          
##                  95% CI : (0.3349, 0.3644)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2506          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.63636       1.00000      0.9448          0.0000
## Specificity                  0.99810       0.99873      0.9825          1.0000
## Pos Pred Value               0.97297       0.96894      0.8800             NaN
## Neg Pred Value               0.96231       1.00000      0.9924          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.06176       0.03824      0.1132          0.0000
## Detection Prevalence         0.06348       0.03946      0.1287          0.0000
## Balanced Accuracy            0.81723       0.99936      0.9636          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9619        0.000      0.0000
## Specificity                0.2636        1.000      1.0000
## Pos Pred Value             0.1774          NaN         NaN
## Neg Pred Value             0.9767        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1363        0.000      0.0000
## Detection Prevalence       0.7684        0.000      0.0000
## Balanced Accuracy          0.6128        0.500      0.5000
db_tda_pc_5.50.5_n4_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      252      0    6        0     0     0    1
##   BOMBAY          5    156    0        0     0     0    0
##   CALI           31      0  462        0    22     0   10
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ         108      0   21     1063   556   608  779
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3495          
##                  95% CI : (0.3349, 0.3644)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2506          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.63636       1.00000      0.9448          0.0000
## Specificity                  0.99810       0.99873      0.9825          1.0000
## Pos Pred Value               0.97297       0.96894      0.8800             NaN
## Neg Pred Value               0.96231       1.00000      0.9924          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.06176       0.03824      0.1132          0.0000
## Detection Prevalence         0.06348       0.03946      0.1287          0.0000
## Balanced Accuracy            0.81723       0.99936      0.9636          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9619        0.000      0.0000
## Specificity                0.2636        1.000      1.0000
## Pos Pred Value             0.1774          NaN         NaN
## Neg Pred Value             0.9767        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1363        0.000      0.0000
## Detection Prevalence       0.7684        0.000      0.0000
## Balanced Accuracy          0.6128        0.500      0.5000
db_tda_pc_5.50.5_n4_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   3.495098e-01   2.506335e-01   3.348685e-01   3.643669e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##   2.511382e-36            NaN
db_tda_pc_5.50.5_n4_db_svm_cf0_ov_acc<-db_tda_pc_5.50.5_n4_db_svm_cf0$overall[1]
db_tda_pc_5.50.5_n4_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.6363636   0.9980999      0.9729730      0.9623135 0.9729730
## Class: BOMBAY     1.0000000   0.9987258      0.9689441      1.0000000 0.9689441
## Class: CALI       0.9447853   0.9824561      0.8800000      0.9924051 0.8800000
## Class: DERMASON   0.0000000   1.0000000            NaN      0.7394608        NA
## Class: HOROZ      0.9619377   0.2635637      0.1773525      0.9767196 0.1773525
## Class: SEKER      0.0000000   1.0000000            NaN      0.8509804        NA
## Class: SIRA       0.0000000   1.0000000            NaN      0.8063725        NA
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.6363636 0.7694656 0.09705882     0.06176471
## Class: BOMBAY   1.0000000 0.9842271 0.03823529     0.03823529
## Class: CALI     0.9447853 0.9112426 0.11985294     0.11323529
## Class: DERMASON 0.0000000        NA 0.26053922     0.00000000
## Class: HOROZ    0.9619377 0.2994883 0.14166667     0.13627451
## Class: SEKER    0.0000000        NA 0.14901961     0.00000000
## Class: SIRA     0.0000000        NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.06348039         0.8172318
## Class: BOMBAY             0.03946078         0.9993629
## Class: CALI               0.12867647         0.9636207
## Class: DERMASON           0.00000000         0.5000000
## Class: HOROZ              0.76838235         0.6127507
## Class: SEKER              0.00000000         0.5000000
## Class: SIRA               0.00000000         0.5000000
db_tda_pc_5.50.5_n4_db_svm_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n4_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_svm_n4_3_fold<-(db_svm_fit_re - db_tda_pc_5.50.5_n4_svm_fit_re)
diff_drybean_tda_pca_5.50.5_svm_n4_3_fold
##      Accuracy
## 1 -0.06294097
## 2 -0.03585256
## 3 -0.05671064
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n4_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n4_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n4_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n4_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n4_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n4_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n4_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n4_3_fold
## $winLeft
## [1] 0.9916333
## 
## $winRope
## [1] 0.008366667
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n4_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n4_3_fold
## $left
## [1] 0.9762494
## 
## $rope
## [1] 0.01244765
## 
## $right
## [1] 0.01130295
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_svm_n4_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm_n4_3_fold))
#bf_tda_pca_5.50.5_rf.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_svm_n4_3_fold)
## t = -6.3283, df = 2, p-value = 0.02407
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.08707766 -0.01659179
## sample estimates:
##   mean of x 
## -0.05183473
### Test set diff
diff_drybean_tda_pca_5.50.5_svm.n4_test<-(db_svm_cf_ov_acc - db_tda_pc_5.50.5_n4_db_svm_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_svm.n4_test
## Accuracy 
## 0.577451
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n4_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n4_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n4_test$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n4_test$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n4_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1543667
## 
## $winRight
## [1] 0.8456333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n4_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_svm.n4_test)))

#BayesFactor
#bf_tda_pca_5.50.5_svm.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm.n4_test)) #bf_tda_pca_5.50.5_svm.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n4_test))

##Node5

DryBean_TDA_PC_5.50.5_n5_SvmFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n5.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')
## Warning: model fit failed for Fold1: sigma= 0.1, C=0.25 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 1.0, C=0.25 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma=10.0, C=0.25 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 0.1, C=0.50 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 1.0, C=0.50 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma=10.0, C=0.50 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 0.1, C=0.75 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 1.0, C=0.75 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma=10.0, C=0.75 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 0.1, C=1.00 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 1.0, C=1.00 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma=10.0, C=1.00 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 0.1, C=1.25 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma= 1.0, C=1.25 Error in indexes[[j]] : subscript out of bounds
## Warning: model fit failed for Fold1: sigma=10.0, C=1.25 Error in indexes[[j]] : subscript out of bounds
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo,
## : There were missing values in resampled performance measures.
DryBean_TDA_PC_5.50.5_n5_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 417 samples
##  16 predictor
##   2 classes: 'BOMBAY', 'CALI' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 277, 278, 279 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy  Kappa
##    0.1   0.25  1         NaN  
##    0.1   0.50  1         NaN  
##    0.1   0.75  1         NaN  
##    0.1   1.00  1         NaN  
##    0.1   1.25  1         NaN  
##    1.0   0.25  1         NaN  
##    1.0   0.50  1         NaN  
##    1.0   0.75  1         NaN  
##    1.0   1.00  1         NaN  
##    1.0   1.25  1         NaN  
##   10.0   0.25  1         NaN  
##   10.0   0.50  1         NaN  
##   10.0   0.75  1         NaN  
##   10.0   1.00  1         NaN  
##   10.0   1.25  1         NaN  
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 10 and C = 0.25.
DryBean_TDA_PC_5.50.5_n5_SvmFit0$resample
##   Accuracy Kappa Resample
## 1       NA    NA    Fold1
## 2        1    NA    Fold2
## 3        1    NA    Fold3
db_tda_pc_5.50.5_n5_svm_fit_re<-DryBean_TDA_PC_5.50.5_n5_SvmFit0 $resample[1]

summary(DryBean_TDA_PC_5.50.5_n5_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_PC_5.50.5_n5_SvmFit0,25) + ggtitle("Adult_TDA_PCA_5.50.5_n5_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_PC_5.50.5_n5_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n5_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n5_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n5_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY        396    156  489     1063   578   608  790
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.0382          
##                  95% CI : (0.0326, 0.0446)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000      0.0000          0.0000
## Specificity                  1.00000       0.00000      1.0000          1.0000
## Pos Pred Value                   NaN       0.03824         NaN             NaN
## Neg Pred Value               0.90294           NaN      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.03824      0.0000          0.0000
## Detection Prevalence         0.00000       1.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n5_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY        396    156  489     1063   578   608  790
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.0382          
##                  95% CI : (0.0326, 0.0446)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000      0.0000          0.0000
## Specificity                  1.00000       0.00000      1.0000          1.0000
## Pos Pred Value                   NaN       0.03824         NaN             NaN
## Neg Pred Value               0.90294           NaN      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.03824      0.0000          0.0000
## Detection Prevalence         0.00000       1.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n5_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##     0.03823529     0.00000000     0.03256139     0.04458199     0.26053922 
## AccuracyPValue  McnemarPValue 
##     1.00000000            NaN
db_tda_pc_5.50.5_n5_db_svm_cf0_ov_acc<-db_tda_pc_5.50.5_n5_db_svm_cf0$overall[1]
db_tda_pc_5.50.5_n5_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value
## Class: BARBUNYA           0           1            NaN      0.9029412
## Class: BOMBAY             1           0     0.03823529            NaN
## Class: CALI               0           1            NaN      0.8801471
## Class: DERMASON           0           1            NaN      0.7394608
## Class: HOROZ              0           1            NaN      0.8583333
## Class: SEKER              0           1            NaN      0.8509804
## Class: SIRA               0           1            NaN      0.8063725
##                  Precision Recall         F1 Prevalence Detection Rate
## Class: BARBUNYA         NA      0         NA 0.09705882     0.00000000
## Class: BOMBAY   0.03823529      1 0.07365439 0.03823529     0.03823529
## Class: CALI             NA      0         NA 0.11985294     0.00000000
## Class: DERMASON         NA      0         NA 0.26053922     0.00000000
## Class: HOROZ            NA      0         NA 0.14166667     0.00000000
## Class: SEKER            NA      0         NA 0.14901961     0.00000000
## Class: SIRA             NA      0         NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA                    0               0.5
## Class: BOMBAY                      1               0.5
## Class: CALI                        0               0.5
## Class: DERMASON                    0               0.5
## Class: HOROZ                       0               0.5
## Class: SEKER                       0               0.5
## Class: SIRA                        0               0.5
db_tda_pc_5.50.5_n5_db_svm_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n5_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_svm_n5_3_fold<-(db_svm_fit_re - db_tda_pc_5.50.5_n5_svm_fit_re)
diff_drybean_tda_pca_5.50.5_svm_n5_3_fold
##      Accuracy
## 1          NA
## 2 -0.06798867
## 3 -0.06801008
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n5_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n5_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n5_3_fold
## $probLeft
## [1] NA
## 
## $probRope
## [1] NA
## 
## $probRight
## [1] NA
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n5_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n5_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n5_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n5_3_fold_odds.left
## [1] NA
# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_svm.n5_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n5_3_fold),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_svm.n5_3_fold

# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n5_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n5_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n5_3_fold
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_svm_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_rf.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm_n5_3_fold))
#bf_tda_pca_5.50.5_rf.n5_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm_n5_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_svm_n5_3_fold)
## t = -6353, df = 1, p-value = 0.0001002
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.06813537 -0.06786337
## sample estimates:
##   mean of x 
## -0.06799937
### Test set diff
diff_drybean_tda_pca_5.50.5_svm.n5_test<-(db_svm_cf_ov_acc - db_tda_pc_5.50.5_n5_db_svm_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_svm.n5_test
##  Accuracy 
## 0.8887255
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_svm.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n5_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_svm.n5_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_svm.n5_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_svm.n5_test$probLeft/bst_dbf_db_tda_pca_5.50.5_svm.n5_test$probRight
bst_dbf_db_tda_pca_5.50.5_svm.n5_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_svm.n5_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n5_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_svm.n5_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1587333
## 
## $winRight
## [1] 0.8412667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_svm.n5_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n5_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_svm.n5_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_svm.n5_test)))

#BayesFactor
#bf_tda_pca_5.50.5_svm.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_svm.n5_test)) #bf_tda_pca_5.50.5_svm.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_svm.n5_test))


#With TDA KDE filter 5 intervals, 50% overlap, 5 bins 
##Node1


DryBean_TDA_KDE_5.50.5_n1_SvmFit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n1.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_KDE_5.50.5_n1_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 8473 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5650, 5648, 5648 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa    
##    0.1   0.25  0.9523182  0.9430179
##    0.1   0.50  0.9532625  0.9441534
##    0.1   0.75  0.9534985  0.9444374
##    0.1   1.00  0.9542066  0.9452842
##    0.1   1.25  0.9550329  0.9462702
##    1.0   0.25  0.9261169  0.9113623
##    1.0   0.50  0.9378005  0.9255079
##    1.0   0.75  0.9437018  0.9326311
##    1.0   1.00  0.9457080  0.9350491
##    1.0   1.25  0.9460622  0.9354778
##   10.0   0.25  0.3028453  0.1127524
##   10.0   0.50  0.4405774  0.2959555
##   10.0   0.75  0.5591906  0.4501030
##   10.0   1.00  0.6531363  0.5705731
##   10.0   1.25  0.6768589  0.6009978
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
DryBean_TDA_KDE_5.50.5_n1_SvmFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9518243 0.9424457    Fold1
## 2 0.9582301 0.9500843    Fold3
## 3 0.9550442 0.9462807    Fold2
ad_tda_kde_5.50.5_n1_svm_fit_re<-DryBean_TDA_KDE_5.50.5_n1_SvmFit0 $resample[1]

summary(DryBean_TDA_PC_5.50.5_n1_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_KDE_5.50.5_n1_SvmFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n1_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n1_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n1_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n1_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
ad_tda_kde_5.50.5_n1_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   10        0     1     3    3
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           17      0  469        0     7     0    1
##   DERMASON        0      0    0      861     4     7   28
##   HOROZ           3      0    4        1   558     0    9
##   SEKER           3      0    1       19     0   577    4
##   SIRA            9      0    5      182     8    21  745
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9142          
##                  95% CI : (0.9052, 0.9226)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8966          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000      0.9591          0.8100
## Specificity                  0.99539       1.00000      0.9930          0.9871
## Pos Pred Value               0.95538       1.00000      0.9494          0.9567
## Neg Pred Value               0.99135       1.00000      0.9944          0.9365
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.03824      0.1150          0.2110
## Detection Prevalence         0.09338       0.03824      0.1211          0.2206
## Balanced Accuracy            0.95729       1.00000      0.9761          0.8985
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9654       0.9490      0.9430
## Specificity                0.9951       0.9922      0.9316
## Pos Pred Value             0.9704       0.9553      0.7680
## Neg Pred Value             0.9943       0.9911      0.9855
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1368       0.1414      0.1826
## Detection Prevalence       0.1409       0.1480      0.2377
## Balanced Accuracy          0.9803       0.9706      0.9373
ad_tda_kde_5.50.5_n1_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   10        0     1     3    3
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           17      0  469        0     7     0    1
##   DERMASON        0      0    0      861     4     7   28
##   HOROZ           3      0    4        1   558     0    9
##   SEKER           3      0    1       19     0   577    4
##   SIRA            9      0    5      182     8    21  745
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9142          
##                  95% CI : (0.9052, 0.9226)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8966          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000      0.9591          0.8100
## Specificity                  0.99539       1.00000      0.9930          0.9871
## Pos Pred Value               0.95538       1.00000      0.9494          0.9567
## Neg Pred Value               0.99135       1.00000      0.9944          0.9365
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.03824      0.1150          0.2110
## Detection Prevalence         0.09338       0.03824      0.1211          0.2206
## Balanced Accuracy            0.95729       1.00000      0.9761          0.8985
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9654       0.9490      0.9430
## Specificity                0.9951       0.9922      0.9316
## Pos Pred Value             0.9704       0.9553      0.7680
## Neg Pred Value             0.9943       0.9911      0.9855
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1368       0.1414      0.1826
## Detection Prevalence       0.1409       0.1480      0.2377
## Balanced Accuracy          0.9803       0.9706      0.9373
ad_tda_kde_5.50.5_n1_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.9142157      0.8965745      0.9052005      0.9226324      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
ad_tda_kde_5.50.5_n1_db_svm_cf0_ov_acc<-ad_tda_kde_5.50.5_n1_db_svm_cf0$overall[1]
ad_tda_kde_5.50.5_n1_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9191919   0.9953855      0.9553806      0.9913490 0.9553806
## Class: BOMBAY     1.0000000   1.0000000      1.0000000      1.0000000 1.0000000
## Class: CALI       0.9591002   0.9930382      0.9493927      0.9944228 0.9493927
## Class: DERMASON   0.8099718   0.9870733      0.9566667      0.9364780 0.9566667
## Class: HOROZ      0.9653979   0.9951456      0.9704348      0.9942939 0.9704348
## Class: SEKER      0.9490132   0.9922235      0.9552980      0.9910817 0.9552980
## Class: SIRA       0.9430380   0.9316109      0.7680412      0.9855305 0.7680412
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9191919 0.9369369 0.09705882     0.08921569
## Class: BOMBAY   1.0000000 1.0000000 0.03823529     0.03823529
## Class: CALI     0.9591002 0.9542218 0.11985294     0.11495098
## Class: DERMASON 0.8099718 0.8772287 0.26053922     0.21102941
## Class: HOROZ    0.9653979 0.9679098 0.14166667     0.13676471
## Class: SEKER    0.9490132 0.9521452 0.14901961     0.14142157
## Class: SIRA     0.9430380 0.8465909 0.19362745     0.18259804
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09338235         0.9572887
## Class: BOMBAY             0.03823529         1.0000000
## Class: CALI               0.12107843         0.9760692
## Class: DERMASON           0.22058824         0.8985225
## Class: HOROZ              0.14093137         0.9802718
## Class: SEKER              0.14803922         0.9706183
## Class: SIRA               0.23774510         0.9373245
ad_tda_kde_5.50.5_n1_db_svm_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n1_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_svm_n1_3_fold<-(db_svm_fit_re - ad_tda_kde_5.50.5_n1_svm_fit_re)
diff_drybean_tda_kde_5.50.5_svm_n1_3_fold
##      Accuracy
## 1 -0.02797282
## 2 -0.02621876
## 3 -0.02305432
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n1_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n1_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n1_3_fold_odds.left<-bst_tda_kde_5.50.5_svm.n1_3_fold$probLeft/bst_tda_kde_5.50.5_svm.n1_3_fold$probRight
bst_tda_kde_5.50.5_svm.n1_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n1_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n1_3_fold
## $winLeft
## [1] 0.9913667
## 
## $winRope
## [1] 0.008633333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n1_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n1_3_fold
## $left
## [1] 0.9945239
## 
## $rope
## [1] 0.004399082
## 
## $right
## [1] 0.00107699
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_svm_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm_n1_3_fold))
#bf_tda_kde_5.50.5_svm.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_svm_n1_3_fold)
## t = -17.891, df = 2, p-value = 0.003109
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.03194090 -0.01955637
## sample estimates:
##   mean of x 
## -0.02574863
### Test set diff
diff_drybean_tda_kde_5.50.5_svm.n1_test<-(db_svm_cf_ov_acc-ad_tda_kde_5.50.5_n1_db_svm_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_svm.n1_test
##  Accuracy 
## 0.0127451
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n1_test),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n1_test_odds.left<-bst_tda_kde_5.50.5_svm.n1_test$probLeft/bst_tda_kde_5.50.5_svm.n1_test$probRight
bst_tda_kde_5.50.5_svm.n1_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n1_test),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.4621333
## 
## $winRight
## [1] 0.5378667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n1_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_svm.n1_test))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm.n1_test)) #bf_tda_kde_5.50.5_svm.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n1_test))


##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node2

DryBean_TDA_KDE_5.50.5_n2_SvmFit0 <- train(as.factor(Class) ~ ., data =  tda.m_kde_dry_bean_dataset_5.50.5.n2.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_KDE_5.50.5_n2_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 7582 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5055, 5055, 5054 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa    
##    0.1   0.25  0.9422319  0.9260614
##    0.1   0.50  0.9450013  0.9296485
##    0.1   0.75  0.9453971  0.9301714
##    0.1   1.00  0.9459246  0.9308563
##    0.1   1.25  0.9465842  0.9317000
##    1.0   0.25  0.9268006  0.9064155
##    1.0   0.50  0.9322083  0.9134002
##    1.0   0.75  0.9360331  0.9182953
##    1.0   1.00  0.9373520  0.9199733
##    1.0   1.25  0.9377475  0.9204803
##   10.0   0.25  0.5129282  0.3219701
##   10.0   0.50  0.6027430  0.4545282
##   10.0   0.75  0.6351886  0.5017353
##   10.0   1.00  0.6717231  0.5542406
##   10.0   1.25  0.6916382  0.5830410
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
DryBean_TDA_KDE_5.50.5_n2_SvmFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9469727 0.9322224    Fold1
## 2 0.9450158 0.9296774    Fold3
## 3 0.9477641 0.9332001    Fold2
ad_tda_kde_5.50.5_n2_svm_fit_re<-DryBean_TDA_KDE_5.50.5_n2_SvmFit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n2_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_KDE_5.50.5_n2_SvmFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n2_SvmFit TDA-Assited Svm")
# Predict outcome using DryBean_TDA_KDE_5.50.5_n2_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n2_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n2_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
ad_tda_kde_5.50.5_n2_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      341      0   12        0     1     0    3
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           26      0  461        0    21     0    1
##   DERMASON        0      0    0      934     6     6   57
##   HOROZ          16    156    8        1   543     0    8
##   SEKER           3      0    1       18     0   581    3
##   SIRA           10      0    7      110     7    21  718
## 
## Overall Statistics
##                                           
##                Accuracy : 0.877           
##                  95% CI : (0.8665, 0.8869)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8506          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.86111       0.00000      0.9427          0.8786
## Specificity                  0.99566       1.00000      0.9866          0.9771
## Pos Pred Value               0.95518           NaN      0.9057          0.9312
## Neg Pred Value               0.98523       0.96176      0.9922          0.9581
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08358       0.00000      0.1130          0.2289
## Detection Prevalence         0.08750       0.00000      0.1248          0.2458
## Balanced Accuracy            0.92838       0.50000      0.9647          0.9279
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9394       0.9556      0.9089
## Specificity                0.9460       0.9928      0.9529
## Pos Pred Value             0.7418       0.9587      0.8225
## Neg Pred Value             0.9895       0.9922      0.9775
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1331       0.1424      0.1760
## Detection Prevalence       0.1794       0.1485      0.2140
## Balanced Accuracy          0.9427       0.9742      0.9309
ad_tda_kde_5.50.5_n2_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      341      0   12        0     1     0    3
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           26      0  461        0    21     0    1
##   DERMASON        0      0    0      934     6     6   57
##   HOROZ          16    156    8        1   543     0    8
##   SEKER           3      0    1       18     0   581    3
##   SIRA           10      0    7      110     7    21  718
## 
## Overall Statistics
##                                           
##                Accuracy : 0.877           
##                  95% CI : (0.8665, 0.8869)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8506          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.86111       0.00000      0.9427          0.8786
## Specificity                  0.99566       1.00000      0.9866          0.9771
## Pos Pred Value               0.95518           NaN      0.9057          0.9312
## Neg Pred Value               0.98523       0.96176      0.9922          0.9581
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08358       0.00000      0.1130          0.2289
## Detection Prevalence         0.08750       0.00000      0.1248          0.2458
## Balanced Accuracy            0.92838       0.50000      0.9647          0.9279
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9394       0.9556      0.9089
## Specificity                0.9460       0.9928      0.9529
## Pos Pred Value             0.7418       0.9587      0.8225
## Neg Pred Value             0.9895       0.9922      0.9775
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1331       0.1424      0.1760
## Detection Prevalence       0.1794       0.1485      0.2140
## Balanced Accuracy          0.9427       0.9742      0.9309
ad_tda_kde_5.50.5_n2_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.8769608      0.8505944      0.8664883      0.8868901      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
ad_tda_kde_5.50.5_n2_db_svm_cf0_ov_acc<-ad_tda_kde_5.50.5_n2_db_svm_cf0$overall[1]
ad_tda_kde_5.50.5_n2_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.8611111   0.9956569      0.9551821      0.9852270 0.9551821
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.9427403   0.9866332      0.9056974      0.9921591 0.9056974
## Class: DERMASON   0.8786453   0.9771296      0.9312064      0.9580760 0.9312064
## Class: HOROZ      0.9394464   0.9460308      0.7418033      0.9895460 0.7418033
## Class: SEKER      0.9555921   0.9927995      0.9587459      0.9922280 0.9587459
## Class: SIRA       0.9088608   0.9528875      0.8224513      0.9775491 0.8224513
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.8611111 0.9057105 0.09705882     0.08357843
## Class: BOMBAY   0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.9427403 0.9238477 0.11985294     0.11299020
## Class: DERMASON 0.8786453 0.9041626 0.26053922     0.22892157
## Class: HOROZ    0.9394464 0.8290076 0.14166667     0.13308824
## Class: SEKER    0.9555921 0.9571664 0.14901961     0.14240196
## Class: SIRA     0.9088608 0.8634997 0.19362745     0.17598039
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA            0.0875000         0.9283840
## Class: BOMBAY              0.0000000         0.5000000
## Class: CALI                0.1247549         0.9646868
## Class: DERMASON            0.2458333         0.9278875
## Class: HOROZ               0.1794118         0.9427386
## Class: SEKER               0.1485294         0.9741958
## Class: SIRA                0.2139706         0.9308741
ad_tda_kde_5.50.5_n2_db_svm_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n2_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_svm_n2_3_fold<-(db_svm_fit_re - ad_tda_kde_5.50.5_n2_svm_fit_re)
diff_drybean_tda_kde_5.50.5_svm_n2_3_fold
##      Accuracy
## 1 -0.02312122
## 2 -0.01300449
## 3 -0.01577422
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n2_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n2_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n2_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n2_3_fold_odds.left<-bst_tda_kde_5.50.5_svm.n2_3_fold$probLeft/bst_tda_kde_5.50.5_svm.n2_3_fold$probRight
bst_tda_kde_5.50.5_svm.n2_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n2_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n2_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n2_3_fold
## $winLeft
## [1] 0.9056667
## 
## $winRope
## [1] 0.09433333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n2_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n2_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n2_3_fold
## $left
## [1] 0.9143819
## 
## $rope
## [1] 0.07766221
## 
## $right
## [1] 0.007955856
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_svm_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm_n2_3_fold))
#bf_tda_kde_5.50.5_svm.n2_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n2_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_svm_n2_3_fold)
## t = -5.7314, df = 2, p-value = 0.02912
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.03028728 -0.00431267
## sample estimates:
##   mean of x 
## -0.01729998
### Test set diff
diff_drybean_tda_kde_5.50.5_svm.n2_test<-(db_svm_cf_ov_acc-ad_tda_kde_5.50.5_n2_db_svm_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_svm.n2_test
## Accuracy 
##     0.05
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n2_test),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n2_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n2_test_odds.left<-bst_tda_kde_5.50.5_svm.n2_test$probLeft/bst_tda_kde_5.50.5_svm.n2_test$probRight
bst_tda_kde_5.50.5_svm.n2_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n2_test),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1578333
## 
## $winRight
## [1] 0.8421667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n2_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n2_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n2_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_svm.n2_test))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm.n2_test)) #bf_tda_kde_5.50.5_svm.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n2_test))

##Node3

DryBean_TDA_KDE_5.50.5_n3_SvmFit0 <- train(as.factor(Class) ~ ., data =  tda.m_kde_dry_bean_dataset_5.50.5.n3.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_KDE_5.50.5_n3_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 4149 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 2767, 2766, 2765 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa       
##    0.1   0.25  0.9113039  0.8656083559
##    0.1   0.50  0.9161243  0.8730287159
##    0.1   0.75  0.9163659  0.8734237721
##    0.1   1.00  0.9170891  0.8745612562
##    0.1   1.25  0.9178120  0.8756815811
##    1.0   0.25  0.8843126  0.8233130802
##    1.0   0.50  0.8939526  0.8383262657
##    1.0   0.75  0.8975674  0.8440411048
##    1.0   1.00  0.8990135  0.8464489877
##    1.0   1.25  0.8992547  0.8469257770
##   10.0   0.25  0.3856352  0.0004507734
##   10.0   0.50  0.4160038  0.0561045848
##   10.0   0.75  0.5049399  0.2087131185
##   10.0   1.00  0.6271389  0.4110371831
##   10.0   1.25  0.6560630  0.4588660759
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
DryBean_TDA_KDE_5.50.5_n3_SvmFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9225760 0.8828739    Fold1
## 2 0.9205202 0.8799162    Fold3
## 3 0.9103398 0.8642546    Fold2
ad_tda_kde_5.50.5_n3_svm_fit_re<-DryBean_TDA_KDE_5.50.5_n3_SvmFit0 $resample[1]

summary(DryBean_TDA_KDE_5.50.5_n3_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_KDE_5.50.5_n3_SvmFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n3_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n3_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n3_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n3_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
ad_tda_kde_5.50.5_n3_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA       34      0    0        0     0     0    2
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      347    156  475      992   184    30   96
##   HOROZ           3      0    9        0   369     0    5
##   SEKER           1      0    0       17     0   553    2
##   SIRA           11      0    5       54    25    25  685
## 
## Overall Statistics
##                                         
##                Accuracy : 0.6453        
##                  95% CI : (0.6304, 0.66)
##     No Information Rate : 0.2605        
##     P-Value [Acc > NIR] : < 2.2e-16     
##                                         
##                   Kappa : 0.5459        
##                                         
##  Mcnemar's Test P-Value : NA            
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                 0.085859       0.00000      0.0000          0.9332
## Specificity                 0.999457       1.00000      1.0000          0.5731
## Pos Pred Value              0.944444           NaN         NaN          0.4351
## Neg Pred Value              0.910485       0.96176      0.8801          0.9606
## Prevalence                  0.097059       0.03824      0.1199          0.2605
## Detection Rate              0.008333       0.00000      0.0000          0.2431
## Detection Prevalence        0.008824       0.00000      0.0000          0.5588
## Balanced Accuracy           0.542658       0.50000      0.5000          0.7531
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity               0.63841       0.9095      0.8671
## Specificity               0.99515       0.9942      0.9635
## Pos Pred Value            0.95596       0.9651      0.8509
## Neg Pred Value            0.94342       0.9843      0.9679
## Prevalence                0.14167       0.1490      0.1936
## Detection Rate            0.09044       0.1355      0.1679
## Detection Prevalence      0.09461       0.1404      0.1973
## Balanced Accuracy         0.81678       0.9519      0.9153
ad_tda_kde_5.50.5_n3_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA       34      0    0        0     0     0    2
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      347    156  475      992   184    30   96
##   HOROZ           3      0    9        0   369     0    5
##   SEKER           1      0    0       17     0   553    2
##   SIRA           11      0    5       54    25    25  685
## 
## Overall Statistics
##                                         
##                Accuracy : 0.6453        
##                  95% CI : (0.6304, 0.66)
##     No Information Rate : 0.2605        
##     P-Value [Acc > NIR] : < 2.2e-16     
##                                         
##                   Kappa : 0.5459        
##                                         
##  Mcnemar's Test P-Value : NA            
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                 0.085859       0.00000      0.0000          0.9332
## Specificity                 0.999457       1.00000      1.0000          0.5731
## Pos Pred Value              0.944444           NaN         NaN          0.4351
## Neg Pred Value              0.910485       0.96176      0.8801          0.9606
## Prevalence                  0.097059       0.03824      0.1199          0.2605
## Detection Rate              0.008333       0.00000      0.0000          0.2431
## Detection Prevalence        0.008824       0.00000      0.0000          0.5588
## Balanced Accuracy           0.542658       0.50000      0.5000          0.7531
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity               0.63841       0.9095      0.8671
## Specificity               0.99515       0.9942      0.9635
## Pos Pred Value            0.95596       0.9651      0.8509
## Neg Pred Value            0.94342       0.9843      0.9679
## Prevalence                0.14167       0.1490      0.1936
## Detection Rate            0.09044       0.1355      0.1679
## Detection Prevalence      0.09461       0.1404      0.1973
## Balanced Accuracy         0.81678       0.9519      0.9153
ad_tda_kde_5.50.5_n3_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.6453431      0.5459017      0.6304409      0.6600370      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
ad_tda_kde_5.50.5_n3_db_svm_cf0_ov_acc<-ad_tda_kde_5.50.5_n3_db_svm_cf0$overall[1]
ad_tda_kde_5.50.5_n3_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA  0.08585859   0.9994571      0.9444444      0.9104847 0.9444444
## Class: BOMBAY    0.00000000   1.0000000            NaN      0.9617647        NA
## Class: CALI      0.00000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON  0.93320790   0.5730858      0.4350877      0.9605556 0.4350877
## Class: HOROZ     0.63840830   0.9951456      0.9559585      0.9434218 0.9559585
## Class: SEKER     0.90953947   0.9942396      0.9650960      0.9843171 0.9650960
## Class: SIRA      0.86708861   0.9635258      0.8509317      0.9679389 0.8509317
##                     Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.08585859 0.1574074 0.09705882    0.008333333
## Class: BOMBAY   0.00000000        NA 0.03823529    0.000000000
## Class: CALI     0.00000000        NA 0.11985294    0.000000000
## Class: DERMASON 0.93320790 0.5934789 0.26053922    0.243137255
## Class: HOROZ    0.63840830 0.7655602 0.14166667    0.090441176
## Class: SEKER    0.90953947 0.9364945 0.14901961    0.135539216
## Class: SIRA     0.86708861 0.8589342 0.19362745    0.167892157
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.008823529         0.5426578
## Class: BOMBAY            0.000000000         0.5000000
## Class: CALI              0.000000000         0.5000000
## Class: DERMASON          0.558823529         0.7531469
## Class: HOROZ             0.094607843         0.8167770
## Class: SEKER             0.140441176         0.9518896
## Class: SIRA              0.197303922         0.9153072
ad_tda_kde_5.50.5_n3_db_svm_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n3_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_svm_n3_3_fold<-(db_svm_fit_re - ad_tda_kde_5.50.5_n3_svm_fit_re)
diff_drybean_tda_kde_5.50.5_svm_n3_3_fold
##      Accuracy
## 1 0.001275502
## 2 0.011491100
## 3 0.021650084
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n3_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n3_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n3_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n3_3_fold_odds.left<-bst_tda_kde_5.50.5_svm.n3_3_fold$probLeft/bst_tda_kde_5.50.5_svm.n3_3_fold$probRight
bst_tda_kde_5.50.5_svm.n3_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n3_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n3_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n3_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.394
## 
## $winRight
## [1] 0.606
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n3_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n3_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n3_3_fold
## $left
## [1] 0.04358046
## 
## $rope
## [1] 0.3806632
## 
## $right
## [1] 0.5757564
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_svm_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm_n3_3_fold))
#bf_tda_kde_5.50.5_svm.n3_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n3_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_svm_n3_3_fold)
## t = 1.9505, df = 2, p-value = 0.1904
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.01383444  0.03677889
## sample estimates:
##  mean of x 
## 0.01147223
### Test set diff
diff_drybean_tda_kde_5.50.5_svm.n3_test<-(db_svm_cf_ov_acc-ad_tda_kde_5.50.5_n3_db_svm_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_svm.n3_test
##  Accuracy 
## 0.2816176
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n3_test),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n3_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n3_test_odds.left<-bst_tda_kde_5.50.5_svm.n3_test$probLeft/bst_tda_kde_5.50.5_svm.n3_test$probRight
bst_tda_kde_5.50.5_svm.n3_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n3_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n3_test),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n3_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1601
## 
## $winRight
## [1] 0.8399
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n3_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n3_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n3_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_svm.n3_test))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm.n3_test)) #bf_tda_kde_5.50.5_svm.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n3_test))

##Node4

DryBean_TDA_KDE_5.50.5_n4_SvmFit0 <- train(as.factor(Class) ~ ., data =  tda.m_kde_dry_bean_dataset_5.50.5.n4.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_KDE_5.50.5_n4_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 2024 samples
##   16 predictor
##    4 classes: 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1349, 1348, 1351 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa     
##    0.1   0.25  0.8176688  0.68953858
##    0.1   0.50  0.8211227  0.69653284
##    0.1   0.75  0.8240834  0.70269504
##    0.1   1.00  0.8221067  0.70002358
##    0.1   1.25  0.8201343  0.69697728
##    1.0   0.25  0.7346624  0.50733591
##    1.0   0.50  0.7687526  0.58599720
##    1.0   0.75  0.7791253  0.61320332
##    1.0   1.00  0.7781449  0.61638940
##    1.0   1.25  0.7786395  0.61975579
##   10.0   0.25  0.5197634  0.00000000
##   10.0   0.50  0.5197634  0.00000000
##   10.0   0.75  0.5227264  0.00769601
##   10.0   1.00  0.5296466  0.02558484
##   10.0   1.25  0.5424883  0.05896260
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 0.75.
DryBean_TDA_KDE_5.50.5_n4_SvmFit0$resample
##    Accuracy     Kappa Resample
## 1 0.8296296 0.7116606    Fold1
## 2 0.8387574 0.7271691    Fold2
## 3 0.8038633 0.6692555    Fold3
ad_tda_kde_5.50.5_n4_svm_fit_re<-DryBean_TDA_KDE_5.50.5_n4_SvmFit0 $resample[1]

summary(DryBean_TDA_KDE_5.50.5_n4_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_KDE_5.50.5_n4_SvmFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n4_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n4_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n4_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
ad_tda_kde_5.50.5_n4_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      395    156  489     1006   577   170  528
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0       17     0   430    1
##   SIRA            1      0    0       40     1     8  261
## 
## Overall Statistics
##                                           
##                Accuracy : 0.4159          
##                  95% CI : (0.4007, 0.4312)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2282          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9464
## Specificity                  1.00000       1.00000      1.0000          0.2327
## Pos Pred Value                   NaN           NaN         NaN          0.3029
## Neg Pred Value               0.90294       0.96176      0.8801          0.9249
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2466
## Detection Prevalence         0.00000       0.00000      0.0000          0.8140
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5895
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000       0.7072     0.33038
## Specificity                1.0000       0.9948     0.98480
## Pos Pred Value                NaN       0.9598     0.83923
## Neg Pred Value             0.8583       0.9510     0.85964
## Prevalence                 0.1417       0.1490     0.19363
## Detection Rate             0.0000       0.1054     0.06397
## Detection Prevalence       0.0000       0.1098     0.07623
## Balanced Accuracy          0.5000       0.8510     0.65759
ad_tda_kde_5.50.5_n4_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      395    156  489     1006   577   170  528
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0       17     0   430    1
##   SIRA            1      0    0       40     1     8  261
## 
## Overall Statistics
##                                           
##                Accuracy : 0.4159          
##                  95% CI : (0.4007, 0.4312)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2282          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9464
## Specificity                  1.00000       1.00000      1.0000          0.2327
## Pos Pred Value                   NaN           NaN         NaN          0.3029
## Neg Pred Value               0.90294       0.96176      0.8801          0.9249
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2466
## Detection Prevalence         0.00000       0.00000      0.0000          0.8140
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5895
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000       0.7072     0.33038
## Specificity                1.0000       0.9948     0.98480
## Pos Pred Value                NaN       0.9598     0.83923
## Neg Pred Value             0.8583       0.9510     0.85964
## Prevalence                 0.1417       0.1490     0.19363
## Detection Rate             0.0000       0.1054     0.06397
## Detection Prevalence       0.0000       0.1098     0.07623
## Balanced Accuracy          0.5000       0.8510     0.65759
ad_tda_kde_5.50.5_n4_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   4.159314e-01   2.282457e-01   4.007498e-01   4.312335e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##  6.264481e-103            NaN
ad_tda_kde_5.50.5_n4_db_svm_cf0_ov_acc<-ad_tda_kde_5.50.5_n4_db_svm_cf0$overall[1]
ad_tda_kde_5.50.5_n4_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.0000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.0000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON   0.9463782   0.2326815      0.3029208      0.9249012 0.3029208
## Class: HOROZ      0.0000000   1.0000000            NaN      0.8583333        NA
## Class: SEKER      0.7072368   0.9948157      0.9598214      0.9509912 0.9598214
## Class: SIRA       0.3303797   0.9848024      0.8392283      0.8596445 0.8392283
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.0000000        NA 0.09705882     0.00000000
## Class: BOMBAY   0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.0000000        NA 0.11985294     0.00000000
## Class: DERMASON 0.9463782 0.4589416 0.26053922     0.24656863
## Class: HOROZ    0.0000000        NA 0.14166667     0.00000000
## Class: SEKER    0.7072368 0.8143939 0.14901961     0.10539216
## Class: SIRA     0.3303797 0.4741144 0.19362745     0.06397059
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.00000000         0.5000000
## Class: BOMBAY             0.00000000         0.5000000
## Class: CALI               0.00000000         0.5000000
## Class: DERMASON           0.81397059         0.5895298
## Class: HOROZ              0.00000000         0.5000000
## Class: SEKER              0.10980392         0.8510263
## Class: SIRA               0.07622549         0.6575911
ad_tda_kde_5.50.5_n4_db_svm_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n4_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_svm_n4_3_fold<-(db_svm_fit_re - ad_tda_kde_5.50.5_n4_svm_fit_re)
diff_drybean_tda_kde_5.50.5_svm_n4_3_fold
##     Accuracy
## 1 0.09422185
## 2 0.09325393
## 3 0.12812663
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n4_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n4_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n4_3_fold_odds.left<-bst_tda_kde_5.50.5_svm.n4_3_fold$probLeft/bst_tda_kde_5.50.5_svm.n4_3_fold$probRight
bst_tda_kde_5.50.5_svm.n4_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n4_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n4_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.0084
## 
## $winRight
## [1] 0.9916
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n4_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n4_3_fold
## $left
## [1] 0.006476539
## 
## $rope
## [1] 0.002922708
## 
## $right
## [1] 0.9906008
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_svm_n4_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm_n4_3_fold))
#bf_tda_kde_5.50.5_svm.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_svm_n4_3_fold)
## t = 9.1748, df = 2, p-value = 0.01167
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.05586523 0.15453638
## sample estimates:
## mean of x 
## 0.1052008
### Test set diff
diff_drybean_tda_kde_5.50.5_svm.n4_test<-(db_svm_cf_ov_acc-ad_tda_kde_5.50.5_n4_db_svm_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_svm.n4_test
##  Accuracy 
## 0.5110294
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n4_test),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n4_test_odds.left<-bst_tda_kde_5.50.5_svm.n4_test$probLeft/bst_tda_kde_5.50.5_svm.n4_test$probRight
bst_tda_kde_5.50.5_svm.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n4_test),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1603667
## 
## $winRight
## [1] 0.8396333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n4_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_svm.n4_test))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm.n4_test)) #bf_tda_kde_5.50.5_svm.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n4_test))

##Node5

DryBean_TDA_KDE_5.50.5_n5_SvmFit0 <- train(as.factor(Class) ~ ., data =  tda.m_kde_dry_bean_dataset_5.50.5.n5.vec, 
                    Importance = T,
                    method = 'svmRadial', 
                  trControl = fitControl,
                          tuneGrid = svmGrid, preProc = c('center','scale'),
                          metric='Accuracy')

DryBean_TDA_KDE_5.50.5_n5_SvmFit0
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 989 samples
##  16 predictor
##   4 classes: 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## Pre-processing: centered (16), scaled (16) 
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 659, 660, 659 
## Resampling results across tuning parameters:
## 
##   sigma  C     Accuracy   Kappa      
##    0.1   0.25  0.7249762  0.470647847
##    0.1   0.50  0.7492278  0.531246300
##    0.1   0.75  0.7482239  0.532952329
##    0.1   1.00  0.7492340  0.537413770
##    0.1   1.25  0.7542845  0.550066092
##    1.0   0.25  0.6248749  0.187999649
##    1.0   0.50  0.6643087  0.304622052
##    1.0   0.75  0.6835191  0.361151065
##    1.0   1.00  0.6835129  0.380201828
##    1.0   1.25  0.6966381  0.422078321
##   10.0   0.25  0.5621842  0.000000000
##   10.0   0.50  0.5621842  0.000000000
##   10.0   0.75  0.5621842  0.000000000
##   10.0   1.00  0.5621842  0.000000000
##   10.0   1.25  0.5652175  0.009307213
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were sigma = 0.1 and C = 1.25.
DryBean_TDA_KDE_5.50.5_n5_SvmFit0$resample
##    Accuracy     Kappa Resample
## 1 0.7666667 0.5689714    Fold1
## 2 0.7545455 0.5571864    Fold3
## 3 0.7416413 0.5240405    Fold2
ad_tda_kde_5.50.5_n5_svm_fit_re<-DryBean_TDA_KDE_5.50.5_n5_SvmFit0 $resample[1]

summary(DryBean_TDA_KDE_5.50.5_n5_SvmFit0)
## Length  Class   Mode 
##      1   ksvm     S4
#vip(DryBean_TDA_KDE_5.50.5_n5_SvmFit0,25) + ggtitle("DryBean_TDA_KDE_5.50.5_n5_SvmFit TDA-Assited Svm")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n5_SvmFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n5_SvmFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
ad_tda_kde_5.50.5_n5_db_svm_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
ad_tda_kde_5.50.5_n5_db_svm_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      396    156  489     1020   577   400  675
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0       11     0   202    0
##   SIRA            0      0    0       32     1     6  115
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3277          
##                  95% CI : (0.3133, 0.3423)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.101           
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9595
## Specificity                  1.00000       1.00000      1.0000          0.1074
## Pos Pred Value                   NaN           NaN         NaN          0.2747
## Neg Pred Value               0.90294       0.96176      0.8801          0.8828
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2500
## Detection Prevalence         0.00000       0.00000      0.0000          0.9100
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5335
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000      0.33224     0.14557
## Specificity                1.0000      0.99683     0.98815
## Pos Pred Value                NaN      0.94836     0.74675
## Neg Pred Value             0.8583      0.89501     0.82807
## Prevalence                 0.1417      0.14902     0.19363
## Detection Rate             0.0000      0.04951     0.02819
## Detection Prevalence       0.0000      0.05221     0.03775
## Balanced Accuracy          0.5000      0.66453     0.56686
ad_tda_kde_5.50.5_n5_db_svm_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      396    156  489     1020   577   400  675
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0       11     0   202    0
##   SIRA            0      0    0       32     1     6  115
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3277          
##                  95% CI : (0.3133, 0.3423)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.101           
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9595
## Specificity                  1.00000       1.00000      1.0000          0.1074
## Pos Pred Value                   NaN           NaN         NaN          0.2747
## Neg Pred Value               0.90294       0.96176      0.8801          0.8828
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2500
## Detection Prevalence         0.00000       0.00000      0.0000          0.9100
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5335
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000      0.33224     0.14557
## Specificity                1.0000      0.99683     0.98815
## Pos Pred Value                NaN      0.94836     0.74675
## Neg Pred Value             0.8583      0.89501     0.82807
## Prevalence                 0.1417      0.14902     0.19363
## Detection Rate             0.0000      0.04951     0.02819
## Detection Prevalence       0.0000      0.05221     0.03775
## Balanced Accuracy          0.5000      0.66453     0.56686
ad_tda_kde_5.50.5_n5_db_svm_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   3.276961e-01   1.009676e-01   3.132987e-01   3.423406e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##   8.635197e-22            NaN
ad_tda_kde_5.50.5_n5_db_svm_cf0_ov_acc<-ad_tda_kde_5.50.5_n5_db_svm_cf0$overall[1]
ad_tda_kde_5.50.5_n5_db_svm_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.0000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.0000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON   0.9595484   0.1073914      0.2747105      0.8828338 0.2747105
## Class: HOROZ      0.0000000   1.0000000            NaN      0.8583333        NA
## Class: SEKER      0.3322368   0.9968318      0.9483568      0.8950091 0.9483568
## Class: SIRA       0.1455696   0.9881459      0.7467532      0.8280693 0.7467532
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.0000000        NA 0.09705882     0.00000000
## Class: BOMBAY   0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.0000000        NA 0.11985294     0.00000000
## Class: DERMASON 0.9595484 0.4271357 0.26053922     0.25000000
## Class: HOROZ    0.0000000        NA 0.14166667     0.00000000
## Class: SEKER    0.3322368 0.4920828 0.14901961     0.04950980
## Class: SIRA     0.1455696 0.2436441 0.19362745     0.02818627
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.00000000         0.5000000
## Class: BOMBAY             0.00000000         0.5000000
## Class: CALI               0.00000000         0.5000000
## Class: DERMASON           0.91004902         0.5334699
## Class: HOROZ              0.00000000         0.5000000
## Class: SEKER              0.05220588         0.6645343
## Class: SIRA               0.03774510         0.5668578
ad_tda_kde_5.50.5_n5_db_svm_cf0_pre_rec_f1<-ad_tda_kde_5.50.5_n5_db_svm_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_svm_n5_3_fold<-(db_svm_fit_re - ad_tda_kde_5.50.5_n5_svm_fit_re)
diff_drybean_tda_kde_5.50.5_svm_n5_3_fold
##    Accuracy
## 1 0.1571848
## 2 0.1774659
## 3 0.1903486
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n5_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n5_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n5_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n5_3_fold_odds.left<-bst_tda_kde_5.50.5_svm.n5_3_fold$probLeft/bst_tda_kde_5.50.5_svm.n5_3_fold$probRight
bst_tda_kde_5.50.5_svm.n5_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n5_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n5_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n5_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.008966667
## 
## $winRight
## [1] 0.9910333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n5_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n5_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n5_3_fold
## $left
## [1] 0.001805102
## 
## $rope
## [1] 0.0004609715
## 
## $right
## [1] 0.9977339
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_svm_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm_n5_3_fold))
#bf_tda_kde_5.50.5_svm.n5_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm_n5_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_svm_n5_3_fold)
## t = 18.13, df = 2, p-value = 0.003029
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.1334678 0.2165317
## sample estimates:
## mean of x 
## 0.1749998
### Test set diff
diff_drybean_tda_kde_5.50.5_svm.n5_test<-(db_svm_cf_ov_acc-ad_tda_kde_5.50.5_n5_db_svm_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_svm.n5_test
##  Accuracy 
## 0.5992647
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_svm.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n5_test),-0.01,0.01)
bst_tda_kde_5.50.5_svm.n5_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_svm.n5_test_odds.left<-bst_tda_kde_5.50.5_svm.n5_test$probLeft/bst_tda_kde_5.50.5_svm.n5_test$probRight
bst_tda_kde_5.50.5_svm.n5_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_svm.n5_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n5_test),-0.01,0.01)
bsr_tda_kde_5.50.5_svm.n5_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1564333
## 
## $winRight
## [1] 0.8435667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_svm.n5_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n5_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_svm.n5_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_svm.n5_test))

#BayesFactor
#bf_tda_kde_5.50.5_svm.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_svm.n5_test)) #bf_tda_kde_5.50.5_svm.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_svm.n4_test))


#Non-TDA-Assisted

nn1Grid<-expand.grid(size = c(2,3,5,7), decay = c(0.3,0.5,0.7))
#Neural Network 
dryBeanNn1Fit <- train(as.factor(Class) ~ ., data = Dry_Bean_DatasetTrain, 
                            Importance = T,
                                    method = 'nnet', 
                                    trControl = fitControl,
                                    tuneGrid = nn1Grid,
                                                    metric='Accuracy')
## # weights:  55
## initial  value 14381.499891 
## iter  10 value 11654.878422
## final  value 11654.867658 
## converged
## # weights:  79
## initial  value 13954.918828 
## iter  10 value 11658.024779
## iter  20 value 11593.114330
## iter  30 value 10532.207889
## iter  40 value 10283.884833
## iter  50 value 7754.501277
## iter  60 value 7498.477578
## iter  70 value 7097.931381
## iter  80 value 6714.281624
## iter  90 value 5980.868315
## iter 100 value 5767.764361
## final  value 5767.764361 
## stopped after 100 iterations
## # weights:  127
## initial  value 15206.528540 
## iter  10 value 11870.803581
## iter  20 value 11655.514193
## iter  30 value 11655.450163
## iter  40 value 11631.196435
## iter  50 value 10760.537668
## iter  60 value 10365.593101
## iter  70 value 8937.080582
## iter  80 value 8204.975592
## iter  90 value 7358.785659
## iter 100 value 6701.362613
## final  value 6701.362613 
## stopped after 100 iterations
## # weights:  175
## initial  value 13286.730083 
## iter  10 value 11680.146740
## iter  20 value 11655.227526
## final  value 11655.218856 
## converged
## # weights:  55
## initial  value 13248.623370 
## iter  10 value 11677.676269
## iter  20 value 11177.067941
## iter  30 value 10142.756346
## iter  40 value 9533.707395
## iter  50 value 9511.170843
## iter  60 value 9368.549716
## iter  70 value 9112.272827
## iter  80 value 8052.142597
## iter  90 value 7195.761897
## iter 100 value 6389.738490
## final  value 6389.738490 
## stopped after 100 iterations
## # weights:  79
## initial  value 12618.500019 
## iter  10 value 11656.290354
## iter  20 value 11655.896866
## iter  30 value 11028.819668
## iter  40 value 10692.778766
## iter  50 value 10580.104115
## iter  60 value 9588.831837
## iter  70 value 9083.853303
## iter  80 value 8877.307848
## iter  90 value 8649.548027
## iter 100 value 8589.135377
## final  value 8589.135377 
## stopped after 100 iterations
## # weights:  127
## initial  value 12557.694008 
## iter  10 value 11676.527515
## iter  20 value 11589.902354
## iter  30 value 11487.197145
## iter  40 value 10764.019514
## iter  50 value 10311.908294
## iter  60 value 8614.176076
## iter  70 value 7292.663924
## iter  80 value 5334.693722
## iter  90 value 4617.473909
## iter 100 value 4229.778307
## final  value 4229.778307 
## stopped after 100 iterations
## # weights:  175
## initial  value 13951.466953 
## iter  10 value 11664.503686
## iter  20 value 11654.007382
## iter  30 value 9781.819674
## iter  40 value 9005.104611
## iter  50 value 8750.913134
## iter  60 value 7583.329944
## iter  70 value 6988.274472
## iter  80 value 6839.051589
## iter  90 value 6716.240813
## iter 100 value 5482.775766
## final  value 5482.775766 
## stopped after 100 iterations
## # weights:  55
## initial  value 12417.580191 
## iter  10 value 11669.136630
## iter  20 value 11655.772180
## iter  30 value 11508.587504
## iter  40 value 11366.854826
## iter  50 value 11151.706526
## iter  60 value 11092.255422
## iter  70 value 10950.511833
## iter  80 value 10733.143807
## iter  90 value 10712.414021
## iter 100 value 10703.818129
## final  value 10703.818129 
## stopped after 100 iterations
## # weights:  79
## initial  value 13779.154394 
## iter  10 value 11657.498248
## iter  20 value 11074.637359
## iter  30 value 9414.582853
## iter  40 value 8844.186004
## iter  50 value 8592.434280
## iter  60 value 8404.480606
## iter  70 value 8152.389331
## iter  80 value 8102.460795
## iter  90 value 7583.569279
## iter 100 value 6791.588951
## final  value 6791.588951 
## stopped after 100 iterations
## # weights:  127
## initial  value 13415.416000 
## iter  10 value 11657.424549
## iter  20 value 11655.379221
## iter  30 value 11655.163758
## iter  40 value 11654.925699
## final  value 11654.922575 
## converged
## # weights:  175
## initial  value 13590.414173 
## iter  10 value 11655.058426
## iter  20 value 11654.901368
## iter  30 value 9774.304635
## iter  40 value 9342.921751
## iter  50 value 9224.001780
## iter  60 value 8992.371996
## iter  70 value 7962.682061
## iter  80 value 7273.385903
## iter  90 value 7089.651309
## iter 100 value 6734.843636
## final  value 6734.843636 
## stopped after 100 iterations
## # weights:  55
## initial  value 13115.576105 
## iter  10 value 11657.167604
## iter  20 value 10968.561794
## iter  30 value 10292.360687
## iter  40 value 9261.587631
## iter  50 value 7840.337705
## iter  60 value 7015.055920
## iter  70 value 6678.699638
## iter  80 value 6093.633016
## iter  90 value 5495.839193
## iter 100 value 5392.100169
## final  value 5392.100169 
## stopped after 100 iterations
## # weights:  79
## initial  value 13275.747067 
## iter  10 value 11657.330695
## final  value 11656.990589 
## converged
## # weights:  127
## initial  value 15017.520089 
## iter  10 value 11656.935101
## final  value 11656.934882 
## converged
## # weights:  175
## initial  value 13502.670617 
## iter  10 value 11657.040715
## iter  20 value 11655.631599
## iter  20 value 11655.631487
## iter  30 value 11433.411930
## iter  40 value 11395.305527
## iter  50 value 11388.528113
## iter  60 value 10089.307659
## iter  70 value 9785.962916
## iter  80 value 8899.492790
## iter  90 value 8742.089500
## iter 100 value 8703.726457
## final  value 8703.726457 
## stopped after 100 iterations
## # weights:  55
## initial  value 12215.159457 
## iter  10 value 11658.333553
## iter  20 value 11646.129051
## iter  30 value 9857.041891
## iter  40 value 9470.943949
## iter  50 value 8615.932731
## iter  60 value 8064.960716
## iter  70 value 7837.799899
## iter  80 value 7472.960803
## iter  90 value 7055.443348
## iter 100 value 6828.698642
## final  value 6828.698642 
## stopped after 100 iterations
## # weights:  79
## initial  value 12996.067957 
## iter  10 value 11665.950407
## iter  20 value 11657.253830
## iter  30 value 11657.140565
## final  value 11657.139252 
## converged
## # weights:  127
## initial  value 12302.186601 
## iter  10 value 11660.789717
## iter  20 value 11657.624723
## iter  30 value 11607.657716
## iter  40 value 11003.840530
## iter  50 value 10837.980447
## iter  60 value 10590.204814
## iter  70 value 10030.588648
## iter  80 value 8837.725183
## iter  90 value 8342.075079
## iter 100 value 7839.172144
## final  value 7839.172144 
## stopped after 100 iterations
## # weights:  175
## initial  value 12935.519038 
## iter  10 value 11376.664909
## iter  20 value 10359.335841
## iter  30 value 9089.111676
## iter  40 value 8737.015286
## iter  50 value 8677.405050
## iter  60 value 7568.373198
## iter  70 value 7412.116397
## iter  80 value 6985.874266
## iter  90 value 6887.547524
## iter 100 value 6829.672350
## final  value 6829.672350 
## stopped after 100 iterations
## # weights:  55
## initial  value 12609.733759 
## iter  10 value 11658.831358
## iter  20 value 10254.131588
## iter  30 value 10126.286785
## iter  40 value 10078.256609
## iter  50 value 10019.529517
## iter  60 value 9986.144781
## final  value 9975.862831 
## converged
## # weights:  79
## initial  value 12832.486592 
## iter  10 value 11682.830808
## final  value 11657.287941 
## converged
## # weights:  127
## initial  value 18274.661880 
## iter  10 value 11738.793043
## iter  20 value 11658.355078
## iter  30 value 10139.244822
## iter  40 value 9035.254868
## iter  50 value 8593.689987
## iter  60 value 8246.535074
## iter  70 value 8043.386442
## iter  80 value 7897.369561
## iter  90 value 7489.492726
## iter 100 value 7209.193773
## final  value 7209.193773 
## stopped after 100 iterations
## # weights:  175
## initial  value 13903.517617 
## iter  10 value 11671.760580
## iter  20 value 11657.509604
## iter  30 value 11657.290174
## final  value 11657.288140 
## converged
## # weights:  55
## initial  value 15344.925451 
## iter  10 value 11693.835490
## iter  20 value 11659.513960
## iter  30 value 11032.757223
## iter  40 value 10796.974804
## iter  50 value 10753.907202
## iter  60 value 10716.116637
## iter  70 value 10466.769551
## iter  80 value 8851.623071
## iter  90 value 6597.787585
## iter 100 value 6360.056445
## final  value 6360.056445 
## stopped after 100 iterations
## # weights:  79
## initial  value 12203.359891 
## iter  10 value 11688.003336
## iter  20 value 11659.542838
## iter  30 value 11147.990696
## iter  40 value 10700.342014
## iter  50 value 10028.156638
## iter  60 value 9246.224076
## iter  70 value 8375.221409
## iter  80 value 7427.580750
## iter  90 value 6781.929680
## iter 100 value 5365.301377
## final  value 5365.301377 
## stopped after 100 iterations
## # weights:  127
## initial  value 15178.295692 
## iter  10 value 11787.994268
## iter  20 value 11659.713952
## iter  30 value 11658.969958
## iter  30 value 11658.969929
## iter  30 value 11658.969817
## final  value 11658.969817 
## converged
## # weights:  175
## initial  value 15573.083516 
## iter  10 value 11783.876871
## iter  20 value 11272.961614
## iter  30 value 9002.217396
## iter  40 value 8901.417888
## iter  50 value 8392.003332
## iter  60 value 7521.789049
## iter  70 value 6360.603822
## iter  80 value 5786.129495
## iter  90 value 5498.048810
## iter 100 value 4973.120595
## final  value 4973.120595 
## stopped after 100 iterations
## # weights:  55
## initial  value 12013.574905 
## iter  10 value 11668.772616
## iter  20 value 11655.080512
## iter  30 value 10851.696619
## iter  40 value 10323.750808
## iter  50 value 8734.819267
## iter  60 value 8532.713078
## iter  70 value 8471.685866
## iter  80 value 8284.631796
## iter  90 value 6597.283020
## iter 100 value 5097.134754
## final  value 5097.134754 
## stopped after 100 iterations
## # weights:  79
## initial  value 13651.791329 
## iter  10 value 11665.874518
## iter  20 value 11659.263896
## iter  30 value 11659.108410
## iter  40 value 10214.208258
## iter  50 value 9929.416188
## iter  60 value 9749.931163
## iter  70 value 9258.185412
## iter  80 value 9055.524153
## iter  90 value 8750.318558
## iter 100 value 8351.209424
## final  value 8351.209424 
## stopped after 100 iterations
## # weights:  127
## initial  value 12909.373007 
## iter  10 value 11664.194883
## iter  20 value 11659.135143
## iter  30 value 11659.081599
## iter  40 value 11563.367156
## iter  50 value 10967.423138
## iter  60 value 9855.017847
## iter  70 value 9582.123202
## iter  80 value 8453.411696
## iter  90 value 8368.701710
## iter 100 value 8317.968660
## final  value 8317.968660 
## stopped after 100 iterations
## # weights:  175
## initial  value 12961.940992 
## iter  10 value 11663.929692
## iter  10 value 11663.929618
## iter  20 value 11661.074598
## iter  30 value 11327.264936
## iter  40 value 10535.846453
## iter  50 value 10017.905173
## iter  60 value 8930.639048
## iter  70 value 8272.818289
## iter  80 value 8218.580658
## iter  90 value 7666.977106
## iter 100 value 7063.258615
## final  value 7063.258615 
## stopped after 100 iterations
## # weights:  55
## initial  value 13851.354348 
## iter  10 value 11677.821182
## iter  20 value 11661.288894
## iter  30 value 11612.804883
## iter  40 value 9673.142277
## iter  50 value 9168.805815
## iter  60 value 8168.576176
## iter  70 value 7076.154833
## iter  80 value 5889.272526
## iter  90 value 5793.180004
## iter 100 value 5411.723020
## final  value 5411.723020 
## stopped after 100 iterations
## # weights:  79
## initial  value 14079.154110 
## iter  10 value 11746.915859
## iter  20 value 11656.406454
## iter  30 value 11092.043448
## iter  40 value 10800.008619
## iter  50 value 9266.778803
## iter  60 value 8790.602969
## iter  70 value 8571.328929
## iter  80 value 8358.772422
## iter  90 value 8166.052165
## iter 100 value 8053.344176
## final  value 8053.344176 
## stopped after 100 iterations
## # weights:  127
## initial  value 13262.985242 
## iter  10 value 11659.374176
## final  value 11659.192371 
## converged
## # weights:  175
## initial  value 14400.233092 
## iter  10 value 11766.267424
## iter  20 value 11604.522865
## iter  30 value 11270.014206
## iter  40 value 10232.276697
## iter  50 value 9388.611955
## iter  60 value 8773.033610
## iter  70 value 7627.954185
## iter  80 value 7404.640214
## iter  90 value 5856.326119
## iter 100 value 5076.978649
## final  value 5076.978649 
## stopped after 100 iterations
## # weights:  55
## initial  value 19525.104548 
## iter  10 value 17488.829220
## iter  20 value 16837.277549
## iter  30 value 15399.635445
## iter  40 value 13724.063716
## iter  50 value 13475.957473
## iter  60 value 12891.857172
## iter  70 value 11581.421930
## iter  80 value 10834.335539
## iter  90 value 10384.890659
## iter 100 value 10367.355463
## final  value 10367.355463 
## stopped after 100 iterations
dryBeanNn1Fit
## Neural Network 
## 
## 9531 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6353, 6354, 6355 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   2     0.3    0.4861303  0.3375035
##   2     0.5    0.6020500  0.5083878
##   2     0.7    0.4295806  0.2673673
##   3     0.3    0.5459169  0.4127435
##   3     0.5    0.3694305  0.1846085
##   3     0.7    0.3857824  0.2053809
##   5     0.3    0.3630985  0.1554784
##   5     0.5    0.5563549  0.4406289
##   5     0.7    0.3484419  0.1399181
##   7     0.3    0.4478486  0.2802325
##   7     0.5    0.5782216  0.4778784
##   7     0.7    0.5055201  0.3676189
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 2 and decay = 0.5.
dryBeanNn1Fit$resample
##    Accuracy     Kappa Resample
## 1 0.7056045 0.6355485    Fold3
## 2 0.5332074 0.4213065    Fold2
## 3 0.5673379 0.4683083    Fold1
db_nn1_fit_re<-dryBeanNn1Fit$resample[1]

summary(dryBeanNn1Fit)
## a 16-2-7 network with 55 weights
## options were - softmax modelling  decay=0.5
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##   -0.28    0.00   -0.01    0.36    0.42   -4.40    7.02    0.00   -0.66   -1.53 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##   -0.68   -0.83   -0.14   -0.04   -0.01   -1.06   -0.66 
##  b->o1 h1->o1 h2->o1 
##   4.96   0.09 -11.66 
##  b->o2 h1->o2 h2->o2 
##   4.20   0.21 -17.78 
##  b->o3 h1->o3 h2->o3 
##   3.70   0.06  -3.23 
##  b->o4 h1->o4 h2->o4 
##  -0.91  -0.48   7.36 
##  b->o5 h1->o5 h2->o5 
## -16.43   0.16  24.05 
##  b->o6 h1->o6 h2->o6 
##   3.43   0.50  -3.03 
##  b->o7 h1->o7 h2->o7 
##   1.03  -0.54   4.27
#vip(dryBeanNn1Fit,25) + ggtitle("non-TDA-Assited NN")

# Predict outcome using model from training data based on testing data
predictions <- predict(dryBeanNn1Fit, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_nn1_cf<-confusionMatrix(data=predictions, as.factor(Dry_Bean_DatasetTest$Class))
db_nn1_cf
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      308    156  174        0     0    19    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    1        0     0     0    0
##   DERMASON        4      0   32      919    41     3  484
##   HOROZ           0      0    3       49   527     0   20
##   SEKER          77      0  241        5     3   508   61
##   SIRA            7      0   38       90     7    78  225
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6098          
##                  95% CI : (0.5946, 0.6248)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.5213          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.77778       0.00000   0.0020450          0.8645
## Specificity                  0.90527       1.00000   1.0000000          0.8131
## Pos Pred Value               0.46880           NaN   1.0000000          0.6197
## Neg Pred Value               0.97429       0.96176   0.8803628          0.9446
## Prevalence                   0.09706       0.03824   0.1198529          0.2605
## Detection Rate               0.07549       0.00000   0.0002451          0.2252
## Detection Prevalence         0.16103       0.00000   0.0002451          0.3635
## Balanced Accuracy            0.84152       0.50000   0.5010225          0.8388
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9118       0.8355     0.28481
## Specificity                0.9794       0.8885     0.93313
## Pos Pred Value             0.8798       0.5676     0.50562
## Neg Pred Value             0.9853       0.9686     0.84457
## Prevalence                 0.1417       0.1490     0.19363
## Detection Rate             0.1292       0.1245     0.05515
## Detection Prevalence       0.1468       0.2194     0.10907
## Balanced Accuracy          0.9456       0.8620     0.60897
db_nn1_cf$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.6098039      0.5212516      0.5946394      0.6248110      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_nn1_cf_ov_acc<-db_nn1_cf$overall[1]
db_nn1_cf$byClass 
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA  0.77777778   0.9052660      0.4687976      0.9742916 0.4687976
## Class: BOMBAY    0.00000000   1.0000000            NaN      0.9617647        NA
## Class: CALI      0.00204499   1.0000000      1.0000000      0.8803628 1.0000000
## Class: DERMASON  0.86453434   0.8130593      0.6196898      0.9445514 0.6196898
## Class: HOROZ     0.91176471   0.9794403      0.8797997      0.9853490 0.8797997
## Class: SEKER     0.83552632   0.8885369      0.5675978      0.9686028 0.5675978
## Class: SIRA      0.28481013   0.9331307      0.5056180      0.8445667 0.5056180
##                     Recall          F1 Prevalence Detection Rate
## Class: BARBUNYA 0.77777778 0.584995252 0.09705882    0.075490196
## Class: BOMBAY   0.00000000          NA 0.03823529    0.000000000
## Class: CALI     0.00204499 0.004081633 0.11985294    0.000245098
## Class: DERMASON 0.86453434 0.721916732 0.26053922    0.225245098
## Class: HOROZ    0.91176471 0.895497026 0.14166667    0.129166667
## Class: SEKER    0.83552632 0.675981371 0.14901961    0.124509804
## Class: SIRA     0.28481013 0.364372470 0.19362745    0.055147059
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.161029412         0.8415219
## Class: BOMBAY            0.000000000         0.5000000
## Class: CALI              0.000245098         0.5010225
## Class: DERMASON          0.363480392         0.8387968
## Class: HOROZ             0.146813725         0.9456025
## Class: SEKER             0.219362745         0.8620316
## Class: SIRA              0.109068627         0.6089704
db_nn1_cf_pre_rec_f1<-db_nn1_cf$byClass[5:7]

##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node1

#Neural Network 1
DryBean_TDA_PC_5.50.5_n1_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n1.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  52
## initial  value 9387.117698 
## iter  10 value 5849.075506
## iter  20 value 5844.997033
## iter  30 value 5826.635205
## iter  40 value 5278.069745
## iter  50 value 4449.761436
## iter  60 value 4381.947117
## iter  70 value 4321.963534
## iter  80 value 4255.482653
## iter  90 value 4215.830160
## iter 100 value 4148.376629
## final  value 4148.376629 
## stopped after 100 iterations
## # weights:  75
## initial  value 9102.360718 
## iter  10 value 5964.402837
## iter  20 value 5847.551152
## iter  30 value 5844.366499
## iter  40 value 5842.794383
## iter  50 value 5842.411745
## iter  60 value 4481.617706
## iter  70 value 3940.091992
## iter  80 value 3798.573930
## iter  90 value 3558.920024
## iter 100 value 2929.779277
## final  value 2929.779277 
## stopped after 100 iterations
## # weights:  121
## initial  value 10717.577008 
## iter  10 value 5882.507082
## iter  20 value 5847.847160
## iter  30 value 5807.585868
## iter  40 value 5140.668782
## iter  50 value 4216.980457
## iter  60 value 2788.065252
## iter  70 value 2374.969830
## iter  80 value 2237.413688
## iter  90 value 2024.863693
## iter 100 value 1715.440075
## final  value 1715.440075 
## stopped after 100 iterations
## # weights:  167
## initial  value 10256.434164 
## iter  10 value 5871.324644
## iter  20 value 5845.218734
## iter  30 value 5841.451211
## iter  40 value 5841.343354
## iter  50 value 5841.178938
## iter  60 value 5838.044347
## iter  70 value 5827.311221
## iter  80 value 5813.127256
## iter  90 value 5342.447321
## iter 100 value 4009.725776
## final  value 4009.725776 
## stopped after 100 iterations
## # weights:  52
## initial  value 8243.357291 
## iter  10 value 5974.967665
## iter  20 value 5853.715008
## iter  30 value 5846.398911
## iter  40 value 5845.905530
## final  value 5845.900460 
## converged
## # weights:  75
## initial  value 9748.981100 
## iter  10 value 5999.739218
## iter  20 value 5853.238185
## iter  30 value 5849.315281
## iter  40 value 5848.764223
## iter  50 value 5845.904781
## final  value 5845.900240 
## converged
## # weights:  121
## initial  value 9996.167283 
## iter  10 value 6008.771215
## iter  20 value 5857.854433
## iter  30 value 5756.375324
## iter  40 value 4096.332066
## iter  50 value 3813.150466
## iter  60 value 3579.681747
## iter  70 value 2645.122775
## iter  80 value 2268.195362
## iter  90 value 2155.113932
## iter 100 value 2031.518687
## final  value 2031.518687 
## stopped after 100 iterations
## # weights:  167
## initial  value 17197.406493 
## iter  10 value 5869.593965
## iter  20 value 5845.359669
## iter  30 value 5845.054597
## iter  40 value 5514.254930
## iter  50 value 4414.102307
## iter  60 value 4166.676859
## iter  70 value 4057.263781
## iter  80 value 3808.358940
## iter  90 value 3675.037444
## iter 100 value 3613.762015
## final  value 3613.762015 
## stopped after 100 iterations
## # weights:  52
## initial  value 10122.966841 
## iter  10 value 5862.316443
## iter  20 value 5848.416875
## final  value 5848.383646 
## converged
## # weights:  75
## initial  value 7806.850901 
## iter  10 value 5869.110961
## iter  20 value 5211.046785
## iter  30 value 4046.699316
## iter  40 value 3853.732001
## iter  50 value 3818.179972
## iter  60 value 3563.494616
## iter  70 value 3499.968676
## iter  80 value 3008.125410
## iter  90 value 2743.318048
## iter 100 value 1995.342795
## final  value 1995.342795 
## stopped after 100 iterations
## # weights:  121
## initial  value 14860.450452 
## iter  10 value 5916.701228
## iter  20 value 5634.568305
## iter  30 value 4174.133782
## iter  40 value 3529.408919
## iter  50 value 2969.476789
## iter  60 value 2742.889865
## iter  70 value 2354.535187
## iter  80 value 2124.137692
## iter  90 value 2022.571824
## iter 100 value 1870.040140
## final  value 1870.040140 
## stopped after 100 iterations
## # weights:  167
## initial  value 12076.093540 
## iter  10 value 6003.278141
## iter  20 value 5533.819720
## iter  30 value 4930.291971
## iter  40 value 4373.451267
## iter  50 value 3627.189425
## iter  60 value 2789.788890
## iter  70 value 2396.423561
## iter  80 value 2172.030415
## iter  90 value 2040.568048
## iter 100 value 1870.144773
## final  value 1870.144773 
## stopped after 100 iterations
## # weights:  52
## initial  value 10463.022489 
## iter  10 value 5863.055700
## iter  20 value 5856.588277
## iter  30 value 5851.329770
## iter  40 value 5671.254799
## iter  50 value 5648.558532
## iter  60 value 5591.015616
## iter  70 value 4694.260519
## iter  80 value 4376.218663
## iter  90 value 4108.075762
## iter 100 value 3953.430705
## final  value 3953.430705 
## stopped after 100 iterations
## # weights:  75
## initial  value 9020.658783 
## iter  10 value 5880.067360
## iter  20 value 5866.995881
## iter  30 value 5463.716851
## iter  40 value 4134.003599
## iter  50 value 3971.168538
## iter  60 value 3107.582278
## iter  70 value 2600.072920
## iter  80 value 2130.470590
## iter  90 value 1973.305062
## iter 100 value 1867.966523
## final  value 1867.966523 
## stopped after 100 iterations
## # weights:  121
## initial  value 11099.026415 
## iter  10 value 5869.951432
## iter  20 value 5851.582799
## iter  30 value 5850.445823
## iter  40 value 5850.364981
## iter  50 value 5850.342453
## iter  60 value 5849.904325
## iter  70 value 5417.240781
## iter  80 value 4861.423086
## iter  90 value 4177.206736
## iter 100 value 3878.577201
## final  value 3878.577201 
## stopped after 100 iterations
## # weights:  167
## initial  value 9409.994182 
## iter  10 value 6039.032130
## iter  20 value 5952.826267
## iter  30 value 5771.011198
## iter  40 value 4711.147072
## iter  50 value 4438.077170
## iter  60 value 4420.550102
## iter  70 value 3412.210965
## iter  80 value 3109.320927
## iter  90 value 2968.229430
## iter 100 value 2579.636162
## final  value 2579.636162 
## stopped after 100 iterations
## # weights:  52
## initial  value 9398.585605 
## iter  10 value 5892.126452
## iter  20 value 5871.511930
## iter  30 value 5856.158991
## iter  40 value 4591.062928
## iter  50 value 3970.241625
## iter  60 value 3863.799188
## iter  70 value 3831.470180
## iter  80 value 3784.837543
## iter  90 value 3632.817077
## iter 100 value 3430.006087
## final  value 3430.006087 
## stopped after 100 iterations
## # weights:  75
## initial  value 10122.293192 
## iter  10 value 5980.165358
## iter  20 value 5860.858083
## iter  30 value 5852.701419
## iter  40 value 5852.685667
## final  value 5852.679276 
## converged
## # weights:  121
## initial  value 7652.413716 
## iter  10 value 5927.239489
## iter  20 value 5855.667597
## iter  30 value 5852.316530
## iter  40 value 5851.774045
## final  value 5851.759643 
## converged
## # weights:  167
## initial  value 7238.502817 
## iter  10 value 5859.391850
## iter  20 value 5852.511083
## iter  30 value 5557.860980
## iter  40 value 5314.605536
## iter  50 value 5272.880621
## iter  60 value 3234.555813
## iter  70 value 2845.351643
## iter  80 value 2480.063576
## iter  90 value 2371.849228
## iter 100 value 2259.826501
## final  value 2259.826501 
## stopped after 100 iterations
## # weights:  52
## initial  value 7967.442663 
## iter  10 value 5880.838883
## iter  20 value 5860.212527
## iter  30 value 5856.615845
## iter  40 value 5856.593842
## iter  40 value 5856.593785
## iter  40 value 5856.593777
## final  value 5856.593777 
## converged
## # weights:  75
## initial  value 7129.999437 
## iter  10 value 5861.092505
## iter  20 value 5770.652566
## iter  30 value 5630.482791
## iter  40 value 4085.148452
## iter  50 value 3764.442998
## iter  60 value 3720.422597
## iter  70 value 3532.402406
## iter  80 value 3516.498569
## iter  90 value 3457.720035
## iter 100 value 2773.940172
## final  value 2773.940172 
## stopped after 100 iterations
## # weights:  121
## initial  value 10508.343255 
## iter  10 value 5879.895374
## iter  20 value 5860.493780
## iter  30 value 5854.327573
## iter  40 value 5504.010261
## iter  50 value 4496.402024
## iter  60 value 4364.764264
## iter  70 value 4089.055035
## iter  80 value 3599.115623
## iter  90 value 2991.640084
## iter 100 value 2175.543339
## final  value 2175.543339 
## stopped after 100 iterations
## # weights:  167
## initial  value 9326.299622 
## iter  10 value 5876.222807
## iter  20 value 5855.680900
## iter  30 value 5853.230720
## iter  40 value 5840.954305
## iter  50 value 5744.205810
## iter  60 value 4694.572628
## iter  70 value 4440.183537
## iter  80 value 3896.862950
## iter  90 value 3227.326330
## iter 100 value 2506.319883
## final  value 2506.319883 
## stopped after 100 iterations
## # weights:  52
## initial  value 10042.275967 
## iter  10 value 5938.215179
## iter  20 value 5888.299898
## iter  30 value 5407.665836
## iter  40 value 4145.067162
## iter  50 value 3903.933593
## iter  60 value 3816.841032
## iter  70 value 3481.808028
## iter  80 value 3324.788162
## iter  90 value 3155.302254
## iter 100 value 2969.211329
## final  value 2969.211329 
## stopped after 100 iterations
## # weights:  75
## initial  value 11375.337869 
## iter  10 value 5892.898931
## iter  20 value 5842.325330
## iter  30 value 5837.254598
## iter  40 value 5837.058638
## iter  50 value 5150.634522
## iter  60 value 4619.492362
## iter  70 value 4458.514098
## iter  80 value 3922.046604
## iter  90 value 3855.215557
## iter 100 value 3762.086349
## final  value 3762.086349 
## stopped after 100 iterations
## # weights:  121
## initial  value 9791.424989 
## iter  10 value 5888.023934
## iter  20 value 5838.057117
## iter  30 value 5836.619337
## final  value 5835.477850 
## converged
## # weights:  167
## initial  value 9422.685067 
## iter  10 value 5837.949845
## iter  20 value 5837.649895
## iter  30 value 5837.229408
## iter  40 value 5836.231597
## iter  50 value 4601.883378
## iter  60 value 3890.646967
## iter  70 value 3832.115843
## iter  80 value 3670.289950
## iter  90 value 2819.100182
## iter 100 value 2631.744933
## final  value 2631.744933 
## stopped after 100 iterations
## # weights:  52
## initial  value 11811.002851 
## iter  10 value 6071.063202
## iter  20 value 5859.923322
## iter  30 value 5841.999285
## iter  40 value 5839.996110
## iter  50 value 5839.979613
## final  value 5839.979417 
## converged
## # weights:  75
## initial  value 11897.974843 
## iter  10 value 5898.214792
## iter  20 value 5884.900256
## iter  30 value 5852.597077
## iter  40 value 5842.054136
## iter  50 value 5838.620148
## iter  60 value 5838.478931
## iter  70 value 5838.408826
## final  value 5838.408410 
## converged
## # weights:  121
## initial  value 12084.191258 
## iter  10 value 5956.641824
## iter  20 value 5844.898023
## iter  30 value 5839.143234
## iter  40 value 5724.275284
## iter  50 value 4008.188603
## iter  60 value 3763.112719
## iter  70 value 3689.659589
## iter  80 value 3525.794699
## iter  90 value 3170.287365
## iter 100 value 2858.462569
## final  value 2858.462569 
## stopped after 100 iterations
## # weights:  167
## initial  value 12123.659143 
## iter  10 value 5857.038343
## iter  20 value 5839.768283
## iter  30 value 5837.700417
## iter  40 value 4989.288389
## iter  50 value 4648.056741
## iter  60 value 4641.883450
## iter  70 value 4392.959955
## iter  80 value 3821.743112
## iter  90 value 3467.007621
## iter 100 value 2892.949069
## final  value 2892.949069 
## stopped after 100 iterations
## # weights:  52
## initial  value 10306.723599 
## iter  10 value 5954.493774
## iter  20 value 5871.330944
## iter  30 value 5815.655803
## iter  40 value 4719.359348
## iter  50 value 4421.528622
## iter  60 value 4395.730756
## iter  70 value 4353.636160
## iter  80 value 4292.460650
## iter  90 value 3858.151659
## iter 100 value 2405.518666
## final  value 2405.518666 
## stopped after 100 iterations
## # weights:  75
## initial  value 9070.684974 
## iter  10 value 5882.578305
## iter  20 value 5842.973502
## iter  30 value 5842.515243
## iter  40 value 5842.155569
## iter  50 value 5581.728181
## iter  60 value 3868.401887
## iter  70 value 3644.482697
## iter  80 value 3564.396390
## iter  90 value 3514.190388
## iter 100 value 3497.026498
## final  value 3497.026498 
## stopped after 100 iterations
## # weights:  121
## initial  value 11806.784094 
## iter  10 value 5876.274741
## iter  20 value 5840.578567
## iter  30 value 5839.913055
## iter  40 value 5839.640024
## iter  50 value 5565.310952
## iter  60 value 4964.549459
## iter  70 value 4827.487563
## iter  80 value 4645.984119
## iter  90 value 4356.623684
## iter 100 value 3375.563619
## final  value 3375.563619 
## stopped after 100 iterations
## # weights:  167
## initial  value 8788.428376 
## iter  10 value 5879.979379
## iter  20 value 5842.449634
## iter  30 value 5840.262955
## iter  40 value 5839.912462
## iter  50 value 5764.248860
## iter  60 value 5327.127750
## iter  70 value 4325.710224
## iter  80 value 3993.068630
## iter  90 value 3783.579745
## iter 100 value 3657.245612
## final  value 3657.245612 
## stopped after 100 iterations
## # weights:  121
## initial  value 15044.822863 
## iter  10 value 8789.111098
## iter  20 value 8778.141506
## iter  30 value 8766.295711
## iter  40 value 8765.799133
## iter  50 value 8745.783781
## iter  60 value 8720.463895
## iter  70 value 8697.304162
## iter  80 value 7598.477232
## iter  90 value 6831.987381
## iter 100 value 6599.726231
## final  value 6599.726231 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n1_NN1Fit0
## Neural Network 
## 
## 7839 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5226, 5228, 5224 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   2     0.3    0.7133311  0.5360189
##   2     0.5    0.5574222  0.2113577
##   2     0.7    0.5962668  0.2744519
##   3     0.3    0.8052540  0.6987787
##   3     0.5    0.4520986  0.0000000
##   3     0.7    0.8048615  0.6877067
##   5     0.3    0.6710616  0.4395232
##   5     0.5    0.7258828  0.5260170
##   5     0.7    0.8644165  0.7915330
##   7     0.3    0.7893953  0.6746336
##   7     0.5    0.7853187  0.6669141
##   7     0.7    0.8378873  0.7536258
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 5 and decay = 0.7.
DryBean_TDA_PC_5.50.5_n1_NN1Fit0$resample
##    Accuracy     Kappa Resample
## 1 0.8897819 0.8296020    Fold1
## 2 0.8114723 0.7134838    Fold3
## 3 0.8919954 0.8315131    Fold2
db_tda_pc_5.50.5_n1_nn1_fit_re<-DryBean_TDA_PC_5.50.5_n1_NN1Fit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n1_NN1Fit0)
## a 16-5-6 network with 121 weights
## options were - softmax modelling  decay=0.7
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.00    0.13    0.00    0.00    0.00    0.00    0.00    0.13    0.00    0.00 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h4  i1->h4  i2->h4  i3->h4  i4->h4  i5->h4  i6->h4  i7->h4  i8->h4  i9->h4 
##    0.00    0.08    0.00    0.00    0.00    0.00    0.00    0.08    0.00    0.00 
## i10->h4 i11->h4 i12->h4 i13->h4 i14->h4 i15->h4 i16->h4 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h5  i1->h5  i2->h5  i3->h5  i4->h5  i5->h5  i6->h5  i7->h5  i8->h5  i9->h5 
##   -0.01   -1.04   -7.46   -2.19    1.85   -0.06   -0.05    1.19   -1.27    0.00 
## i10->h5 i11->h5 i12->h5 i13->h5 i14->h5 i15->h5 i16->h5 
##   -0.01    0.00    0.01    0.00    0.00    0.03   -0.02 
##  b->o1 h1->o1 h2->o1 h3->o1 h4->o1 h5->o1 
##  -1.19  -1.19   0.10  -1.19  -1.19   3.55 
##  b->o2 h1->o2 h2->o2 h3->o2 h4->o2 h5->o2 
##  -0.72  -0.72  -0.02  -0.72  -0.72   0.26 
##  b->o3 h1->o3 h2->o3 h3->o3 h4->o3 h5->o3 
##   1.40   1.40   0.01   1.40   1.40  -3.70 
##  b->o4 h1->o4 h2->o4 h3->o4 h4->o4 h5->o4 
##  -0.06  -0.06   0.04  -0.06  -0.06  -3.30 
##  b->o5 h1->o5 h2->o5 h3->o5 h4->o5 h5->o5 
##   0.69   0.69  -0.08   0.68   0.69  -0.07 
##  b->o6 h1->o6 h2->o6 h3->o6 h4->o6 h5->o6 
##  -0.12  -0.12  -0.05  -0.12  -0.12   3.25
#vip(DryBean_TDA_PC_5.50.5_n1_NN1Fit0,25) + ggtitle("dryBean_TDA_PCA_5.50.5_n1_NN1Fit TDA-Assited NN")


# Predict outcome using DryBean_TDA_PC_5.50.5_n1_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n1_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n1_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n1_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0      754    17    34   10
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        3     0     1    0
##   SIRA          396    156  489      306   561   573  780
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3762          
##                  95% CI : (0.3613, 0.3913)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2134          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.7093
## Specificity                  1.00000       1.00000      1.0000          0.9798
## Pos Pred Value                   NaN           NaN         NaN          0.9252
## Neg Pred Value               0.90294       0.96176      0.8801          0.9054
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.1848
## Detection Prevalence         0.00000       0.00000      0.0000          0.1998
## Balanced Accuracy            0.50000       0.50000      0.5000          0.8445
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000    0.0016447      0.9873
## Specificity                1.0000    0.9991359      0.2459
## Pos Pred Value                NaN    0.2500000      0.2392
## Neg Pred Value             0.8583    0.8510795      0.9878
## Prevalence                 0.1417    0.1490196      0.1936
## Detection Rate             0.0000    0.0002451      0.1912
## Detection Prevalence       0.0000    0.0009804      0.7993
## Balanced Accuracy          0.5000    0.5003903      0.6166
db_tda_pc_5.50.5_n1_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0      754    17    34   10
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        3     0     1    0
##   SIRA          396    156  489      306   561   573  780
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3762          
##                  95% CI : (0.3613, 0.3913)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2134          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.7093
## Specificity                  1.00000       1.00000      1.0000          0.9798
## Pos Pred Value                   NaN           NaN         NaN          0.9252
## Neg Pred Value               0.90294       0.96176      0.8801          0.9054
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.1848
## Detection Prevalence         0.00000       0.00000      0.0000          0.1998
## Balanced Accuracy            0.50000       0.50000      0.5000          0.8445
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000    0.0016447      0.9873
## Specificity                1.0000    0.9991359      0.2459
## Pos Pred Value                NaN    0.2500000      0.2392
## Neg Pred Value             0.8583    0.8510795      0.9878
## Prevalence                 0.1417    0.1490196      0.1936
## Detection Rate             0.0000    0.0002451      0.1912
## Detection Prevalence       0.0000    0.0009804      0.7993
## Balanced Accuracy          0.5000    0.5003903      0.6166
db_tda_pc_5.50.5_n1_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   3.762255e-01   2.134490e-01   3.613313e-01   3.912971e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##   3.855318e-59            NaN
db_tda_pc_5.50.5_n1_db_nn1_cf0_ov_acc<-db_tda_pc_5.50.5_n1_db_nn1_cf0$overall[1]
db_tda_pc_5.50.5_n1_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA 0.000000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY   0.000000000   1.0000000            NaN      0.9617647        NA
## Class: CALI     0.000000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON 0.709313264   0.9797812      0.9251534      0.9053599 0.9251534
## Class: HOROZ    0.000000000   1.0000000            NaN      0.8583333        NA
## Class: SEKER    0.001644737   0.9991359      0.2500000      0.8510795 0.2500000
## Class: SIRA     0.987341772   0.2458967      0.2391904      0.9877900 0.2391904
##                      Recall          F1 Prevalence Detection Rate
## Class: BARBUNYA 0.000000000          NA 0.09705882    0.000000000
## Class: BOMBAY   0.000000000          NA 0.03823529    0.000000000
## Class: CALI     0.000000000          NA 0.11985294    0.000000000
## Class: DERMASON 0.709313264 0.802981896 0.26053922    0.184803922
## Class: HOROZ    0.000000000          NA 0.14166667    0.000000000
## Class: SEKER    0.001644737 0.003267974 0.14901961    0.000245098
## Class: SIRA     0.987341772 0.385090101 0.19362745    0.191176471
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA         0.0000000000         0.5000000
## Class: BOMBAY           0.0000000000         0.5000000
## Class: CALI             0.0000000000         0.5000000
## Class: DERMASON         0.1997549020         0.8445473
## Class: HOROZ            0.0000000000         0.5000000
## Class: SEKER            0.0009803922         0.5003903
## Class: SIRA             0.7992647059         0.6166192
db_tda_pc_5.50.5_n1_db_nn1_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n1_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold<-(db_nn1_fit_re - db_tda_pc_5.50.5_n1_nn1_fit_re)
diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold
##     Accuracy
## 1 -0.1841773
## 2 -0.2782648
## 3 -0.3246575
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold
## $winLeft
## [1] 0.9911
## 
## $winRope
## [1] 0.0089
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n1_3_fold
## $left
## [1] 0.983029
## 
## $rope
## [1] 0.002296291
## 
## $right
## [1] 0.01467474
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold))
#bf_tda_pca_5.50.5_nn1.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n1_3_fold)
## t = -6.3489, df = 2, p-value = 0.02392
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.44017309 -0.08455999
## sample estimates:
##  mean of x 
## -0.2623665
### Test set diff
diff_drybean_tda_pca_5.50.5_nn1.n1_test<-(db_nn1_cf_ov_acc - db_tda_pc_5.50.5_n1_db_nn1_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_nn1.n1_test
##  Accuracy 
## 0.2335784
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n1_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n1_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n1_test$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n1_test$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n1_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n1_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1570333
## 
## $winRight
## [1] 0.8429667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n1_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nn1.n1_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n1_test)) #bf_tda_pca_5.50.5_nn1.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n1_test))

##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node2

##DryBean_TDA_PC_5.50.5_n2_NN1Fit0 <- nnet(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n2.vec, size=2, range = 0.6,, type='class')

#Neural Network 1
DryBean_TDA_PC_5.50.5_n2_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n2.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  55
## initial  value 12736.191538 
## iter  10 value 10610.958280
## iter  20 value 10603.664209
## iter  30 value 10594.123255
## iter  40 value 9026.794862
## iter  50 value 8543.245991
## iter  60 value 8286.361469
## iter  70 value 8130.579930
## iter  80 value 8045.208899
## iter  90 value 8003.465701
## iter 100 value 7869.928072
## final  value 7869.928072 
## stopped after 100 iterations
## # weights:  79
## initial  value 15223.202922 
## iter  10 value 10710.258688
## iter  20 value 10605.042773
## iter  30 value 10476.472891
## iter  40 value 8753.212144
## iter  50 value 8047.567014
## iter  60 value 7908.533163
## iter  70 value 7801.096546
## iter  80 value 7587.094129
## iter  90 value 7404.869795
## iter 100 value 7333.645771
## final  value 7333.645771 
## stopped after 100 iterations
## # weights:  127
## initial  value 17975.538196 
## iter  10 value 10602.393681
## iter  20 value 10598.065706
## iter  30 value 10597.323422
## iter  40 value 10597.228972
## iter  50 value 10597.138297
## final  value 10597.137996 
## converged
## # weights:  175
## initial  value 14075.783447 
## iter  10 value 10612.085956
## iter  20 value 10604.115645
## iter  30 value 10597.897086
## iter  40 value 9139.633220
## iter  50 value 8795.782568
## iter  60 value 8559.120816
## iter  70 value 8240.052884
## iter  80 value 7186.620864
## iter  90 value 6726.283999
## iter 100 value 6462.181534
## final  value 6462.181534 
## stopped after 100 iterations
## # weights:  55
## initial  value 14698.319479 
## iter  10 value 10924.945070
## iter  20 value 10644.121566
## iter  30 value 10603.955040
## iter  40 value 10603.146437
## final  value 10603.143155 
## converged
## # weights:  79
## initial  value 12488.627179 
## iter  10 value 10705.099613
## iter  20 value 10611.796473
## iter  30 value 10601.303556
## iter  40 value 10600.676717
## final  value 10600.660746 
## converged
## # weights:  127
## initial  value 15086.492630 
## iter  10 value 10647.902160
## iter  20 value 10603.203200
## iter  30 value 10366.728790
## iter  40 value 8419.139641
## iter  50 value 7703.566279
## iter  60 value 7504.043825
## iter  70 value 7152.511085
## iter  80 value 6898.527692
## iter  90 value 6514.681298
## iter 100 value 6331.364532
## final  value 6331.364532 
## stopped after 100 iterations
## # weights:  175
## initial  value 14859.020836 
## iter  10 value 10632.374701
## iter  20 value 9153.199996
## iter  30 value 8542.551332
## iter  40 value 8500.232819
## iter  50 value 7702.286424
## iter  60 value 6301.716804
## iter  70 value 6161.730002
## iter  80 value 5918.295249
## iter  90 value 5577.952333
## iter 100 value 5041.921312
## final  value 5041.921312 
## stopped after 100 iterations
## # weights:  55
## initial  value 13923.153351 
## iter  10 value 10668.275834
## iter  20 value 10605.907297
## final  value 10605.901142 
## converged
## # weights:  79
## initial  value 14147.593684 
## iter  10 value 10643.817657
## final  value 10602.661509 
## converged
## # weights:  127
## initial  value 14315.155410 
## iter  10 value 10662.049478
## iter  20 value 10602.478466
## iter  30 value 10431.233024
## iter  40 value 8346.444445
## iter  50 value 8214.412817
## iter  60 value 7885.296122
## iter  70 value 7122.146826
## iter  80 value 6734.549797
## iter  90 value 6074.458837
## iter 100 value 5854.471652
## final  value 5854.471652 
## stopped after 100 iterations
## # weights:  175
## initial  value 18164.868195 
## iter  10 value 10730.845897
## iter  20 value 10608.253897
## iter  30 value 10599.486946
## iter  40 value 10579.713690
## iter  50 value 9471.201045
## iter  60 value 8524.548241
## iter  70 value 7936.473149
## iter  80 value 7857.725430
## iter  90 value 7801.447906
## iter 100 value 7322.140276
## final  value 7322.140276 
## stopped after 100 iterations
## # weights:  55
## initial  value 14425.417301 
## iter  10 value 10754.426854
## iter  20 value 10603.068720
## iter  30 value 10469.581909
## iter  40 value 9937.170262
## iter  50 value 8115.574635
## iter  60 value 7914.761603
## iter  70 value 7270.327200
## iter  80 value 6291.700198
## iter  90 value 5151.550613
## iter 100 value 4046.431264
## final  value 4046.431264 
## stopped after 100 iterations
## # weights:  79
## initial  value 14011.124602 
## iter  10 value 10639.958485
## iter  20 value 10612.051107
## iter  30 value 10599.426996
## iter  40 value 10599.332461
## iter  50 value 10599.196640
## final  value 10599.184427 
## converged
## # weights:  127
## initial  value 15100.699472 
## iter  10 value 10619.946660
## iter  20 value 10605.178379
## iter  30 value 10599.380075
## iter  40 value 10597.424856
## iter  50 value 8201.096442
## iter  60 value 8132.392657
## iter  70 value 7991.841543
## iter  80 value 7601.986692
## iter  90 value 6519.010766
## iter 100 value 5171.414284
## final  value 5171.414284 
## stopped after 100 iterations
## # weights:  175
## initial  value 14168.668569 
## iter  10 value 10964.287442
## iter  20 value 10611.986849
## iter  30 value 10546.377277
## iter  40 value 10506.015065
## iter  50 value 10467.759046
## iter  60 value 10236.744915
## iter  70 value 8912.945840
## iter  80 value 6360.964035
## iter  90 value 6087.883597
## iter 100 value 5954.975783
## final  value 5954.975783 
## stopped after 100 iterations
## # weights:  55
## initial  value 12308.107549 
## iter  10 value 10690.081465
## iter  20 value 10609.275796
## iter  30 value 10602.215175
## final  value 10602.199729 
## converged
## # weights:  79
## initial  value 14930.161860 
## iter  10 value 10720.077225
## iter  20 value 10617.375102
## iter  30 value 10604.757070
## iter  40 value 10503.750333
## iter  50 value 9434.723210
## iter  60 value 9123.408338
## iter  70 value 8490.301283
## iter  80 value 7579.851671
## iter  90 value 7105.289410
## iter 100 value 6624.444419
## final  value 6624.444419 
## stopped after 100 iterations
## # weights:  127
## initial  value 15085.188121 
## iter  10 value 10680.810785
## iter  20 value 10610.547047
## iter  30 value 10602.006149
## iter  40 value 10601.843221
## iter  50 value 10601.143243
## iter  60 value 10600.772776
## iter  70 value 10595.179204
## iter  80 value 9297.320251
## iter  90 value 8159.967856
## iter 100 value 7923.121024
## final  value 7923.121024 
## stopped after 100 iterations
## # weights:  175
## initial  value 17107.846923 
## iter  10 value 10610.602014
## iter  20 value 10603.752383
## iter  30 value 10600.923381
## iter  40 value 10599.078948
## iter  50 value 10598.175340
## iter  60 value 10289.616507
## iter  70 value 8426.437341
## iter  80 value 8167.018635
## iter  90 value 8112.926579
## iter 100 value 8097.898047
## final  value 8097.898047 
## stopped after 100 iterations
## # weights:  55
## initial  value 14469.609975 
## iter  10 value 10637.957389
## iter  20 value 10604.263707
## final  value 10604.208496 
## converged
## # weights:  79
## initial  value 12845.036227 
## iter  10 value 10672.422383
## iter  20 value 10613.507709
## iter  30 value 10602.683348
## iter  40 value 10602.606587
## iter  50 value 10602.460917
## final  value 10602.457321 
## converged
## # weights:  127
## initial  value 13106.767814 
## iter  10 value 10668.221736
## iter  20 value 10615.791172
## iter  30 value 10602.947094
## iter  40 value 10601.763132
## iter  50 value 10579.371388
## iter  60 value 9069.789626
## iter  70 value 8983.033983
## iter  80 value 6790.773906
## iter  90 value 6227.468044
## iter 100 value 5791.039611
## final  value 5791.039611 
## stopped after 100 iterations
## # weights:  175
## initial  value 13996.973073 
## iter  10 value 10604.148276
## iter  20 value 10600.647894
## iter  30 value 10600.603039
## iter  40 value 10600.598263
## iter  40 value 10600.598158
## iter  50 value 10600.191994
## iter  60 value 8700.121422
## iter  70 value 8562.131718
## iter  80 value 8338.511242
## iter  90 value 8222.542915
## iter 100 value 7252.649278
## final  value 7252.649278 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  52
## initial  value 11189.484925 
## iter  10 value 10581.174010
## iter  10 value 10581.173960
## iter  10 value 10581.173906
## final  value 10581.173906 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  75
## initial  value 13588.630109 
## iter  10 value 10581.916132
## iter  20 value 10581.182581
## iter  30 value 10581.074803
## iter  40 value 8658.686645
## iter  50 value 8024.496778
## iter  60 value 7933.140620
## iter  70 value 7423.657824
## iter  80 value 6256.298340
## iter  90 value 5689.819030
## iter 100 value 5308.115401
## final  value 5308.115401 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  121
## initial  value 13453.771182 
## iter  10 value 10581.152121
## final  value 10581.041595 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  167
## initial  value 11645.128186 
## iter  10 value 10581.042469
## iter  20 value 9837.678711
## iter  30 value 8885.168951
## iter  40 value 8103.479241
## iter  50 value 7518.083637
## iter  60 value 7425.997854
## iter  70 value 7173.568794
## iter  80 value 6770.507175
## iter  90 value 6579.863858
## iter 100 value 6460.683520
## final  value 6460.683520 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  52
## initial  value 11629.586881 
## iter  10 value 10582.435588
## iter  20 value 10581.471911
## iter  30 value 10581.437337
## iter  40 value 10581.336906
## iter  50 value 10581.216179
## final  value 10581.206452 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  75
## initial  value 13205.845075 
## iter  10 value 10582.953110
## iter  20 value 10581.223361
## iter  30 value 10525.093344
## iter  40 value 8737.007103
## iter  50 value 8167.696441
## iter  60 value 7278.096954
## iter  70 value 6565.414790
## iter  80 value 6094.660487
## iter  90 value 5213.477640
## iter 100 value 5005.826073
## final  value 5005.826073 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  121
## initial  value 13415.109462 
## iter  10 value 10591.527472
## iter  20 value 10581.174091
## iter  30 value 10581.153828
## iter  30 value 10581.153779
## iter  40 value 10581.060825
## final  value 10581.056416 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  167
## initial  value 11967.304456 
## iter  10 value 10602.591394
## iter  20 value 10581.330890
## iter  30 value 10581.088468
## iter  40 value 10533.825874
## iter  50 value 10497.001765
## iter  60 value 9532.608338
## iter  70 value 8437.647957
## iter  80 value 8324.462396
## iter  90 value 8321.409880
## iter 100 value 8295.244109
## final  value 8295.244109 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  52
## initial  value 11316.816399 
## iter  10 value 10587.008746
## iter  20 value 10582.299848
## iter  30 value 10580.837620
## iter  40 value 9703.592902
## iter  50 value 8605.834346
## iter  60 value 8245.387152
## iter  70 value 8099.623639
## iter  80 value 7960.785012
## iter  90 value 7924.541603
## iter 100 value 7883.177412
## final  value 7883.177412 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  75
## initial  value 11748.898205 
## iter  10 value 10581.527250
## iter  20 value 10581.369730
## iter  30 value 8710.491725
## iter  40 value 8340.265604
## iter  50 value 7928.199532
## iter  60 value 7130.627731
## iter  70 value 6683.310983
## iter  80 value 6346.060428
## iter  90 value 5796.672058
## iter 100 value 5502.126886
## final  value 5502.126886 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  121
## initial  value 10804.338800 
## iter  10 value 10581.158227
## final  value 10581.156325 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'BOMBAY' is empty
## # weights:  167
## initial  value 11902.177813 
## iter  10 value 10581.547693
## iter  20 value 10581.516516
## iter  20 value 10581.516476
## iter  30 value 10492.412951
## iter  40 value 10128.484116
## iter  50 value 9632.090284
## iter  60 value 8688.194705
## iter  70 value 7958.138650
## iter  80 value 6707.604316
## iter  90 value 6447.330221
## iter 100 value 6417.920097
## final  value 6417.920097 
## stopped after 100 iterations
## # weights:  175
## initial  value 25196.346574 
## iter  10 value 16163.956115
## iter  20 value 15946.809606
## iter  30 value 15901.223883
## iter  40 value 15889.681445
## iter  50 value 15889.665570
## iter  60 value 15888.704245
## final  value 15888.606791 
## converged
DryBean_TDA_PC_5.50.5_n2_NN1Fit0
## Neural Network 
## 
## 9515 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6344, 6345, 6341 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa      
##   2     0.3    0.4301061   0.32129608
##   2     0.5    0.2044674   0.00000000
##   2     0.7    0.2422746   0.03542725
##   3     0.3    0.2949376   0.10782251
##   3     0.5    0.3369107   0.16718787
##   3     0.7    0.1898696  -0.04066983
##   5     0.3    0.3543097   0.21592629
##   5     0.5    0.3686806   0.25256830
##   5     0.7    0.4161126   0.32071143
##   7     0.3    0.4683291   0.34835435
##   7     0.5    0.4051630   0.28740099
##   7     0.7    0.4004130   0.28022583
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 7 and decay = 0.3.
DryBean_TDA_PC_5.50.5_n2_NN1Fit0$resample
##    Accuracy       Kappa Resample
## 1 0.6602524 0.560514774    Fold2
## 2 0.5963418 0.481550296    Fold1
## 3 0.1483932 0.002997983    Fold3
db_tda_pc_5.50.5_n2_nn1_fit_re<-DryBean_TDA_PC_5.50.5_n2_NN1Fit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n2_NN1Fit0)
## a 16-7-7 network with 175 weights
## options were - softmax modelling  decay=0.3
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.00    0.01    0.00    0.00    0.00    0.00    0.00    0.01    0.00    0.00 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h4  i1->h4  i2->h4  i3->h4  i4->h4  i5->h4  i6->h4  i7->h4  i8->h4  i9->h4 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h4 i11->h4 i12->h4 i13->h4 i14->h4 i15->h4 i16->h4 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h5  i1->h5  i2->h5  i3->h5  i4->h5  i5->h5  i6->h5  i7->h5  i8->h5  i9->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h5 i11->h5 i12->h5 i13->h5 i14->h5 i15->h5 i16->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h6  i1->h6  i2->h6  i3->h6  i4->h6  i5->h6  i6->h6  i7->h6  i8->h6  i9->h6 
##    0.00    0.01    0.00    0.00    0.00    0.00    0.00    0.01    0.00    0.00 
## i10->h6 i11->h6 i12->h6 i13->h6 i14->h6 i15->h6 i16->h6 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h7  i1->h7  i2->h7  i3->h7  i4->h7  i5->h7  i6->h7  i7->h7  i8->h7  i9->h7 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h7 i11->h7 i12->h7 i13->h7 i14->h7 i15->h7 i16->h7 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##  b->o1 h1->o1 h2->o1 h3->o1 h4->o1 h5->o1 h6->o1 h7->o1 
##   0.14   0.00   0.14   0.14   0.14   0.14   0.14   0.00 
##  b->o2 h1->o2 h2->o2 h3->o2 h4->o2 h5->o2 h6->o2 h7->o2 
##  -0.97   0.01  -0.97  -0.97  -0.97  -0.97  -0.97   0.00 
##  b->o3 h1->o3 h2->o3 h3->o3 h4->o3 h5->o3 h6->o3 h7->o3 
##   0.10   0.00   0.10   0.10   0.10   0.10   0.10   0.00 
##  b->o4 h1->o4 h2->o4 h3->o4 h4->o4 h5->o4 h6->o4 h7->o4 
##   0.26  -0.01   0.26   0.26   0.26   0.26   0.26   0.00 
##  b->o5 h1->o5 h2->o5 h3->o5 h4->o5 h5->o5 h6->o5 h7->o5 
##   0.18   0.00   0.18   0.18   0.19   0.18   0.18   0.00 
##  b->o6 h1->o6 h2->o6 h3->o6 h4->o6 h5->o6 h6->o6 h7->o6 
##   0.01   0.00   0.01   0.01   0.02   0.01   0.01   0.00 
##  b->o7 h1->o7 h2->o7 h3->o7 h4->o7 h5->o7 h6->o7 h7->o7 
##   0.27  -0.01   0.27   0.27   0.27   0.27   0.27   0.00
#vip(DryBean_TDA_PC_5.50.5_n2_NN1Fit0,25) + ggtitle("dryBean_TDA_PCA_5.50.5_n2_NN1Fit TDA-Assited NN")


# Predict outcome using DryBean_TDA_PC_5.50.5_n2_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n2_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n2_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
db_tda_pc_5.50.5_n2_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA          396    156  489     1063   578   608  790
## 
## Overall Statistics
##                                           
##                Accuracy : 0.1936          
##                  95% CI : (0.1816, 0.2061)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.0000
## Specificity                  1.00000       1.00000      1.0000          1.0000
## Pos Pred Value                   NaN           NaN         NaN             NaN
## Neg Pred Value               0.90294       0.96176      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.0000
## Detection Prevalence         0.00000       0.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      1.0000
## Specificity                1.0000        1.000      0.0000
## Pos Pred Value                NaN          NaN      0.1936
## Neg Pred Value             0.8583        0.851         NaN
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.1936
## Detection Prevalence       0.0000        0.000      1.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n2_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA          396    156  489     1063   578   608  790
## 
## Overall Statistics
##                                           
##                Accuracy : 0.1936          
##                  95% CI : (0.1816, 0.2061)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.0000
## Specificity                  1.00000       1.00000      1.0000          1.0000
## Pos Pred Value                   NaN           NaN         NaN             NaN
## Neg Pred Value               0.90294       0.96176      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.0000
## Detection Prevalence         0.00000       0.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      1.0000
## Specificity                1.0000        1.000      0.0000
## Pos Pred Value                NaN          NaN      0.1936
## Neg Pred Value             0.8583        0.851         NaN
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.1936
## Detection Prevalence       0.0000        0.000      1.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n2_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.1936275      0.0000000      0.1816029      0.2060923      0.2605392 
## AccuracyPValue  McnemarPValue 
##      1.0000000            NaN
db_tda_pc_5.50.5_n2_db_nn1_cf0_ov_acc<-db_tda_pc_5.50.5_n2_db_nn1_cf0$overall[1]
db_tda_pc_5.50.5_n2_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA           0           1            NaN      0.9029412        NA
## Class: BOMBAY             0           1            NaN      0.9617647        NA
## Class: CALI               0           1            NaN      0.8801471        NA
## Class: DERMASON           0           1            NaN      0.7394608        NA
## Class: HOROZ              0           1            NaN      0.8583333        NA
## Class: SEKER              0           1            NaN      0.8509804        NA
## Class: SIRA               1           0      0.1936275            NaN 0.1936275
##                 Recall        F1 Prevalence Detection Rate Detection Prevalence
## Class: BARBUNYA      0        NA 0.09705882      0.0000000                    0
## Class: BOMBAY        0        NA 0.03823529      0.0000000                    0
## Class: CALI          0        NA 0.11985294      0.0000000                    0
## Class: DERMASON      0        NA 0.26053922      0.0000000                    0
## Class: HOROZ         0        NA 0.14166667      0.0000000                    0
## Class: SEKER         0        NA 0.14901961      0.0000000                    0
## Class: SIRA          1 0.3244353 0.19362745      0.1936275                    1
##                 Balanced Accuracy
## Class: BARBUNYA               0.5
## Class: BOMBAY                 0.5
## Class: CALI                   0.5
## Class: DERMASON               0.5
## Class: HOROZ                  0.5
## Class: SEKER                  0.5
## Class: SIRA                   0.5
db_tda_pc_5.50.5_n2_db_nn1_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n2_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold<-(db_nn1_fit_re - db_tda_pc_5.50.5_n2_nn1_fit_re)
diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold
##      Accuracy
## 1  0.04535217
## 2 -0.06313442
## 3  0.41894475
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold
## $probLeft
## [1] 0.25
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold_odds.left
## [1] 0.5
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold
## $winLeft
## [1] 0.1529333
## 
## $winRope
## [1] 0.046
## 
## $winRight
## [1] 0.8010667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n2_3_fold
## $left
## [1] 0.2418801
## 
## $rope
## [1] 0.02783128
## 
## $right
## [1] 0.7302886
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold))
#bf_tda_pca_5.50.5_nn1.n2_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n2_3_fold)
## t = 0.91583, df = 2, p-value = 0.4564
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.4945099  0.7619516
## sample estimates:
## mean of x 
## 0.1337208
### Test set diff
diff_drybean_tda_pca_5.50.5_nn1.n2_test<-(db_nn1_cf_ov_acc - db_tda_pc_5.50.5_n2_db_nn1_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_nn1.n2_test
##  Accuracy 
## 0.4161765
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n2_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n2_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n2_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n2_test$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n2_test$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n2_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n2_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1598667
## 
## $winRight
## [1] 0.8401333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n2_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n2_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n2_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nn1.n2_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n2_test)) #bf_tda_pca_5.50.5_nn1.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n2_test))


##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node3

#Neural Network 1
DryBean_TDA_PC_5.50.5_n3_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n3.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  55
## initial  value 6722.184776 
## iter  10 value 4880.085496
## iter  20 value 4864.168943
## iter  30 value 4820.612062
## iter  40 value 3711.199166
## iter  50 value 3289.843726
## iter  60 value 3214.266146
## iter  70 value 3183.495580
## iter  80 value 3144.069776
## iter  90 value 3111.158053
## iter 100 value 2825.772144
## final  value 2825.772144 
## stopped after 100 iterations
## # weights:  79
## initial  value 6716.838583 
## iter  10 value 4868.725792
## iter  20 value 4862.207902
## iter  30 value 4821.430972
## iter  40 value 4504.013033
## iter  50 value 4176.593648
## iter  60 value 3498.853614
## iter  70 value 2944.729837
## iter  80 value 2273.817040
## iter  90 value 2189.644863
## iter 100 value 2122.372451
## final  value 2122.372451 
## stopped after 100 iterations
## # weights:  127
## initial  value 7613.496328 
## iter  10 value 4914.879776
## iter  20 value 4872.932557
## iter  30 value 4857.735167
## iter  40 value 4853.071814
## iter  50 value 4816.116692
## iter  60 value 4793.758317
## iter  70 value 3286.240689
## iter  80 value 2699.811847
## iter  90 value 2681.938835
## iter 100 value 2437.476463
## final  value 2437.476463 
## stopped after 100 iterations
## # weights:  175
## initial  value 8319.362306 
## iter  10 value 4887.897632
## iter  20 value 4863.614173
## iter  30 value 4835.620148
## iter  40 value 4549.832362
## iter  50 value 3672.862397
## iter  60 value 3293.454696
## iter  70 value 3149.939440
## iter  80 value 2847.062378
## iter  90 value 2540.411951
## iter 100 value 2356.711175
## final  value 2356.711175 
## stopped after 100 iterations
## # weights:  55
## initial  value 6924.851133 
## iter  10 value 4952.512991
## iter  20 value 4872.642039
## iter  30 value 4864.292511
## iter  40 value 4861.717926
## iter  50 value 4816.470105
## iter  60 value 4748.741424
## iter  70 value 3727.750457
## iter  80 value 3514.077902
## iter  90 value 3172.501580
## iter 100 value 2962.225516
## final  value 2962.225516 
## stopped after 100 iterations
## # weights:  79
## initial  value 6277.988583 
## iter  10 value 4894.206976
## iter  20 value 4877.737113
## iter  30 value 4842.409833
## iter  40 value 4581.382538
## iter  50 value 3931.128364
## iter  60 value 3739.517081
## iter  70 value 3312.755480
## iter  80 value 2722.083974
## iter  90 value 2419.110802
## iter 100 value 2234.926909
## final  value 2234.926909 
## stopped after 100 iterations
## # weights:  127
## initial  value 7033.837268 
## iter  10 value 4883.930115
## iter  20 value 4867.014020
## iter  30 value 4689.979608
## iter  40 value 4470.136068
## iter  50 value 4097.878016
## iter  60 value 3467.013629
## iter  70 value 3253.177569
## iter  80 value 3040.282242
## iter  90 value 2685.106476
## iter 100 value 2492.530175
## final  value 2492.530175 
## stopped after 100 iterations
## # weights:  175
## initial  value 7911.488260 
## iter  10 value 5266.787169
## iter  20 value 4958.521698
## iter  30 value 4857.748285
## iter  40 value 4454.069462
## iter  50 value 3351.050146
## iter  60 value 3099.489960
## iter  70 value 2940.854461
## iter  80 value 2707.588683
## iter  90 value 2679.650324
## iter 100 value 2621.775739
## final  value 2621.775739 
## stopped after 100 iterations
## # weights:  55
## initial  value 7069.548851 
## iter  10 value 5023.577505
## iter  20 value 4895.666388
## iter  30 value 4884.368010
## iter  40 value 4883.960732
## iter  50 value 4883.908047
## iter  60 value 4866.495837
## iter  70 value 3741.589032
## iter  80 value 3573.564991
## iter  90 value 3319.162514
## iter 100 value 3120.743704
## final  value 3120.743704 
## stopped after 100 iterations
## # weights:  79
## initial  value 7361.088884 
## iter  10 value 4968.133912
## iter  20 value 4879.953178
## iter  30 value 4864.812446
## iter  40 value 4859.941803
## iter  50 value 4279.473637
## iter  60 value 3895.750342
## iter  70 value 3598.139677
## iter  80 value 3386.481672
## iter  90 value 2644.524899
## iter 100 value 2433.383792
## final  value 2433.383792 
## stopped after 100 iterations
## # weights:  127
## initial  value 6624.404102 
## iter  10 value 4936.745512
## iter  20 value 4872.995873
## iter  30 value 4549.916082
## iter  40 value 4104.852438
## iter  50 value 3445.176517
## iter  60 value 2781.552432
## iter  70 value 2529.018984
## iter  80 value 2487.946434
## iter  90 value 2449.850524
## iter 100 value 2230.837063
## final  value 2230.837063 
## stopped after 100 iterations
## # weights:  175
## initial  value 10191.401842 
## iter  10 value 4919.607541
## iter  20 value 4806.435993
## iter  30 value 3673.996280
## iter  40 value 3633.991763
## iter  50 value 3346.379626
## iter  60 value 2669.298381
## iter  70 value 2553.856481
## iter  80 value 2540.628710
## iter  90 value 2531.975559
## iter 100 value 2492.519715
## final  value 2492.519715 
## stopped after 100 iterations
## # weights:  55
## initial  value 6872.855300 
## iter  10 value 4971.285214
## iter  20 value 4871.760966
## iter  30 value 4799.614266
## iter  40 value 4587.213291
## iter  50 value 4289.193849
## iter  60 value 3897.765471
## iter  70 value 3290.407648
## iter  80 value 2864.108798
## iter  90 value 2722.930601
## iter 100 value 2659.966331
## final  value 2659.966331 
## stopped after 100 iterations
## # weights:  79
## initial  value 7104.684945 
## iter  10 value 4870.789446
## iter  20 value 4868.893892
## iter  30 value 4626.886827
## iter  40 value 4286.935098
## iter  50 value 3829.002347
## iter  60 value 3091.330690
## iter  70 value 2970.467683
## iter  80 value 2782.485949
## iter  90 value 2404.158297
## iter 100 value 2111.710496
## final  value 2111.710496 
## stopped after 100 iterations
## # weights:  127
## initial  value 7653.600939 
## iter  10 value 4894.447787
## iter  20 value 4869.335920
## iter  30 value 4865.206875
## iter  40 value 4864.432872
## iter  50 value 4863.842286
## iter  60 value 4236.542810
## iter  70 value 4104.278662
## iter  80 value 3831.291856
## iter  90 value 3598.097464
## iter 100 value 3498.525453
## final  value 3498.525453 
## stopped after 100 iterations
## # weights:  175
## initial  value 8495.352048 
## iter  10 value 4960.810730
## iter  20 value 4873.774066
## iter  30 value 4869.660613
## iter  40 value 4107.011162
## iter  50 value 3480.275009
## iter  60 value 3347.434724
## iter  70 value 3256.316366
## iter  80 value 3052.015448
## iter  90 value 2985.581281
## iter 100 value 2913.522040
## final  value 2913.522040 
## stopped after 100 iterations
## # weights:  55
## initial  value 7929.646289 
## iter  10 value 4889.583282
## iter  20 value 4873.265291
## iter  30 value 4833.973414
## iter  40 value 4311.959087
## iter  50 value 3387.806144
## iter  60 value 2638.432141
## iter  70 value 2296.368105
## iter  80 value 2234.966812
## iter  90 value 2203.600098
## iter 100 value 2198.865484
## final  value 2198.865484 
## stopped after 100 iterations
## # weights:  79
## initial  value 7639.107229 
## iter  10 value 4878.923419
## iter  20 value 4874.733280
## iter  30 value 4193.317105
## iter  40 value 3745.557729
## iter  50 value 3225.703409
## iter  60 value 3165.218959
## iter  70 value 3154.072650
## iter  80 value 3067.354891
## iter  90 value 3017.689349
## iter 100 value 2902.639352
## final  value 2902.639352 
## stopped after 100 iterations
## # weights:  127
## initial  value 6246.533951 
## iter  10 value 4885.511667
## iter  20 value 4868.434272
## iter  30 value 4867.322548
## iter  40 value 4865.953825
## iter  50 value 3397.449430
## iter  60 value 2594.975318
## iter  70 value 2454.290663
## iter  80 value 2331.295090
## iter  90 value 2066.900776
## iter 100 value 1848.499644
## final  value 1848.499644 
## stopped after 100 iterations
## # weights:  175
## initial  value 8371.015412 
## iter  10 value 4973.883408
## iter  20 value 4880.599278
## iter  30 value 4876.903572
## iter  40 value 4875.883785
## iter  50 value 4853.518878
## iter  60 value 4577.092601
## iter  70 value 4521.623501
## iter  80 value 4437.556803
## iter  90 value 4372.400446
## iter 100 value 4078.776473
## final  value 4078.776473 
## stopped after 100 iterations
## # weights:  55
## initial  value 6944.237666 
## iter  10 value 4889.293286
## iter  20 value 4874.775707
## iter  30 value 4869.487860
## iter  40 value 4867.640468
## iter  50 value 4867.377780
## iter  60 value 4532.237764
## iter  70 value 3872.712453
## iter  80 value 3670.475708
## iter  90 value 3379.926343
## iter 100 value 3089.101117
## final  value 3089.101117 
## stopped after 100 iterations
## # weights:  79
## initial  value 7780.948790 
## iter  10 value 4885.740287
## iter  20 value 4877.297495
## iter  30 value 3400.374926
## iter  40 value 2995.665489
## iter  50 value 2864.364162
## iter  60 value 2493.250596
## iter  70 value 2295.694386
## iter  80 value 2160.966139
## iter  90 value 1943.938148
## iter 100 value 1728.400159
## final  value 1728.400159 
## stopped after 100 iterations
## # weights:  127
## initial  value 7643.904938 
## iter  10 value 4944.152377
## iter  20 value 4920.938228
## iter  30 value 4912.436515
## iter  40 value 3995.696258
## iter  50 value 3791.611134
## iter  60 value 3731.833309
## iter  70 value 3721.292403
## iter  80 value 3701.340945
## iter  90 value 2976.324524
## iter 100 value 2363.253150
## final  value 2363.253150 
## stopped after 100 iterations
## # weights:  175
## initial  value 6810.211284 
## iter  10 value 4903.172697
## iter  20 value 4870.474604
## iter  30 value 4867.293403
## iter  40 value 4771.266401
## iter  50 value 4685.482735
## iter  60 value 3811.685130
## iter  70 value 3198.771741
## iter  80 value 2759.823472
## iter  90 value 2417.944804
## iter 100 value 2352.905572
## final  value 2352.905572 
## stopped after 100 iterations
## # weights:  55
## initial  value 6990.397583 
## iter  10 value 4898.041724
## iter  20 value 4865.568488
## iter  30 value 4864.171139
## iter  40 value 3535.493099
## iter  50 value 3236.991133
## iter  60 value 3056.723080
## iter  70 value 2949.213652
## iter  80 value 2904.297700
## iter  90 value 2833.328505
## iter 100 value 2813.550727
## final  value 2813.550727 
## stopped after 100 iterations
## # weights:  79
## initial  value 10108.106129 
## iter  10 value 4868.523457
## iter  20 value 4868.211361
## iter  30 value 4782.171557
## iter  40 value 4470.542694
## iter  50 value 3546.052780
## iter  60 value 2467.906855
## iter  70 value 1983.581494
## iter  80 value 1800.802476
## iter  90 value 1230.571966
## iter 100 value 982.099720
## final  value 982.099720 
## stopped after 100 iterations
## # weights:  127
## initial  value 8007.100939 
## iter  10 value 4941.614892
## iter  20 value 4867.700407
## iter  30 value 4858.668404
## iter  40 value 3559.954505
## iter  50 value 3274.611795
## iter  60 value 3162.820968
## iter  70 value 3094.590076
## iter  80 value 2962.692734
## iter  90 value 2760.779170
## iter 100 value 1980.967320
## final  value 1980.967320 
## stopped after 100 iterations
## # weights:  175
## initial  value 10400.892119 
## iter  10 value 4882.899670
## iter  20 value 4872.749279
## iter  30 value 4860.937479
## iter  40 value 4658.323956
## iter  50 value 3343.644161
## iter  60 value 2683.897000
## iter  70 value 2088.557134
## iter  80 value 1807.565505
## iter  90 value 1674.711474
## iter 100 value 1600.459344
## final  value 1600.459344 
## stopped after 100 iterations
## # weights:  55
## initial  value 7146.779859 
## iter  10 value 4882.222525
## iter  20 value 4867.716289
## iter  30 value 4867.453676
## iter  40 value 4864.126562
## iter  50 value 3774.305281
## iter  60 value 3140.019291
## iter  70 value 3111.966855
## iter  80 value 2985.357391
## iter  90 value 2908.167417
## iter 100 value 2898.455001
## final  value 2898.455001 
## stopped after 100 iterations
## # weights:  79
## initial  value 6695.368768 
## iter  10 value 4927.757340
## iter  20 value 4867.027980
## iter  30 value 3674.517072
## iter  40 value 3473.446900
## iter  50 value 3397.032403
## iter  60 value 3040.325660
## iter  70 value 2834.887657
## iter  80 value 2728.477108
## iter  90 value 2665.982385
## iter 100 value 2484.670710
## final  value 2484.670710 
## stopped after 100 iterations
## # weights:  127
## initial  value 8976.823892 
## iter  10 value 5056.350915
## iter  20 value 4884.288482
## iter  30 value 4864.762217
## iter  40 value 4860.622092
## iter  50 value 3817.410277
## iter  60 value 3468.697034
## iter  70 value 3439.478600
## iter  80 value 3174.386930
## iter  90 value 2439.993210
## iter 100 value 2067.535269
## final  value 2067.535269 
## stopped after 100 iterations
## # weights:  175
## initial  value 9238.455739 
## iter  10 value 4906.331768
## iter  20 value 4899.475269
## iter  30 value 4863.361323
## iter  40 value 4862.573773
## iter  50 value 3680.511717
## iter  60 value 3312.440623
## iter  70 value 3280.365848
## iter  80 value 3201.105025
## iter  90 value 3170.534451
## iter 100 value 3031.896870
## final  value 3031.896870 
## stopped after 100 iterations
## # weights:  55
## initial  value 8119.150739 
## iter  10 value 4904.921314
## iter  20 value 4881.243491
## iter  30 value 4878.290697
## iter  40 value 3671.478107
## iter  50 value 3397.464755
## iter  60 value 3362.000649
## iter  70 value 3264.128067
## iter  80 value 2659.302454
## iter  90 value 2242.990524
## iter 100 value 1834.438735
## final  value 1834.438735 
## stopped after 100 iterations
## # weights:  79
## initial  value 7996.221053 
## iter  10 value 4871.294392
## iter  20 value 4870.218512
## iter  30 value 4869.936682
## iter  40 value 4100.173098
## iter  50 value 3111.748279
## iter  60 value 2988.496427
## iter  70 value 2892.548402
## iter  80 value 2628.189098
## iter  90 value 2592.293585
## iter 100 value 2279.461852
## final  value 2279.461852 
## stopped after 100 iterations
## # weights:  127
## initial  value 7567.230997 
## iter  10 value 4961.472935
## iter  20 value 4861.939081
## iter  30 value 4861.059181
## iter  40 value 4860.505769
## iter  50 value 4628.744784
## iter  60 value 4428.023611
## iter  70 value 3406.755882
## iter  80 value 3236.132591
## iter  90 value 2931.776803
## iter 100 value 2906.430831
## final  value 2906.430831 
## stopped after 100 iterations
## # weights:  175
## initial  value 8728.371180 
## iter  10 value 4891.900960
## iter  20 value 4867.671105
## iter  30 value 4864.699468
## iter  40 value 4701.099634
## iter  50 value 4179.862300
## iter  60 value 3884.927597
## iter  70 value 3618.082496
## iter  80 value 3485.101297
## iter  90 value 3317.734232
## iter 100 value 3132.649767
## final  value 3132.649767 
## stopped after 100 iterations
## # weights:  79
## initial  value 9527.464660 
## iter  10 value 7354.808236
## iter  20 value 7302.381537
## iter  30 value 7129.535192
## iter  40 value 5143.308648
## iter  50 value 4679.977705
## iter  60 value 4657.636977
## iter  70 value 4536.169310
## iter  80 value 4266.516622
## iter  90 value 4189.323548
## iter 100 value 4157.348083
## final  value 4157.348083 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n3_NN1Fit0
## Neural Network 
## 
## 5355 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 3569, 3571, 3570 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   2     0.3    0.6425770  0.4691813
##   2     0.5    0.7044234  0.5648260
##   2     0.7    0.7142842  0.5800084
##   3     0.3    0.8558380  0.7925276
##   3     0.5    0.7390937  0.6196016
##   3     0.7    0.7785532  0.6909881
##   5     0.3    0.7467416  0.6291674
##   5     0.5    0.8101019  0.7297981
##   5     0.7    0.7344639  0.6137419
##   7     0.3    0.7706630  0.6685634
##   7     0.5    0.5984667  0.3993136
##   7     0.7    0.6476224  0.5068622
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 3 and decay = 0.3.
DryBean_TDA_PC_5.50.5_n3_NN1Fit0$resample
##    Accuracy     Kappa Resample
## 1 0.9266106 0.8971866    Fold3
## 2 0.8267937 0.7484719    Fold2
## 3 0.8141097 0.7319242    Fold1
db_tda_pc_5.50.5_n3_nn1_fit_re<-DryBean_TDA_PC_5.50.5_n3_NN1Fit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n3_NN1Fit0)
## a 16-3-7 network with 79 weights
## options were - softmax modelling  decay=0.3
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##   -0.02   -1.07   -0.33   -1.54   -1.00   -0.07   -0.01    1.07   -0.25    0.07 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##   -0.02   -0.02   -0.01    0.00    0.00    0.00   -0.02 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.09    0.03   -0.28   -3.83   -0.03    0.09    0.06   -0.02    4.46    0.64 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.09    0.14    0.09    0.00    0.00    0.09    0.13 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##  b->o1 h1->o1 h2->o1 h3->o1 
##  -0.46   1.21   3.07  -0.46 
##  b->o2 h1->o2 h2->o2 h3->o2 
##  -1.16   0.99   2.34  -1.16 
##  b->o3 h1->o3 h2->o3 h3->o3 
##   0.35  -0.11   2.65   0.35 
##  b->o4 h1->o4 h2->o4 h3->o4 
##  -1.45   1.05  -1.65  -1.45 
##  b->o5 h1->o5 h2->o5 h3->o5 
##   2.66  -1.35  -5.03   2.66 
##  b->o6 h1->o6 h2->o6 h3->o6 
##  -1.24   0.00  -1.07  -1.24 
##  b->o7 h1->o7 h2->o7 h3->o7 
##   1.30  -1.79  -0.31   1.30
#vip(DryBean_TDA_PC_5.50.5_n3_NN1Fit0,25) + ggtitle("dryBean_TDA_PCA_5.50.5_n3_NN1Fit TDA-Assited NN")

# Predict outcome using DryBean_TDA_PC_5.50.5_n3_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n3_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n3_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
db_tda_pc_5.50.5_n3_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      312    144  261        8     8    11   34
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           79     12  216      776    10   597  686
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           5      0   12      273   560     0   70
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        6     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.2667          
##                  95% CI : (0.2531, 0.2805)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 0.1909          
##                                           
##                   Kappa : 0.1662          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.78788       0.00000     0.44172          0.0000
## Specificity                  0.87351       1.00000     0.39850          1.0000
## Pos Pred Value               0.40103           NaN     0.09091             NaN
## Neg Pred Value               0.97456       0.96176     0.83979          0.7395
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.07647       0.00000     0.05294          0.0000
## Detection Prevalence         0.19069       0.00000     0.58235          0.0000
## Balanced Accuracy            0.83069       0.50000     0.42011          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9689        0.000    0.000000
## Specificity                0.8972        1.000    0.998176
## Pos Pred Value             0.6087          NaN    0.000000
## Neg Pred Value             0.9943        0.851    0.806087
## Prevalence                 0.1417        0.149    0.193627
## Detection Rate             0.1373        0.000    0.000000
## Detection Prevalence       0.2255        0.000    0.001471
## Balanced Accuracy          0.9330        0.500    0.499088
db_tda_pc_5.50.5_n3_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      312    144  261        8     8    11   34
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           79     12  216      776    10   597  686
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           5      0   12      273   560     0   70
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        6     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.2667          
##                  95% CI : (0.2531, 0.2805)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 0.1909          
##                                           
##                   Kappa : 0.1662          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.78788       0.00000     0.44172          0.0000
## Specificity                  0.87351       1.00000     0.39850          1.0000
## Pos Pred Value               0.40103           NaN     0.09091             NaN
## Neg Pred Value               0.97456       0.96176     0.83979          0.7395
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.07647       0.00000     0.05294          0.0000
## Detection Prevalence         0.19069       0.00000     0.58235          0.0000
## Balanced Accuracy            0.83069       0.50000     0.42011          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9689        0.000    0.000000
## Specificity                0.8972        1.000    0.998176
## Pos Pred Value             0.6087          NaN    0.000000
## Neg Pred Value             0.9943        0.851    0.806087
## Prevalence                 0.1417        0.149    0.193627
## Detection Rate             0.1373        0.000    0.000000
## Detection Prevalence       0.2255        0.000    0.001471
## Balanced Accuracy          0.9330        0.500    0.499088
db_tda_pc_5.50.5_n3_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.2666667      0.1661610      0.2531459      0.2805223      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.1909134            NaN
db_tda_pc_5.50.5_n3_db_nn1_cf0_ov_acc<-db_tda_pc_5.50.5_n3_db_nn1_cf0$overall[1]
db_tda_pc_5.50.5_n3_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value
## Class: BARBUNYA   0.7878788   0.8735071     0.40102828      0.9745609
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647
## Class: CALI       0.4417178   0.3984962     0.09090909      0.8397887
## Class: DERMASON   0.0000000   1.0000000            NaN      0.7394608
## Class: HOROZ      0.9688581   0.8972016     0.60869565      0.9943038
## Class: SEKER      0.0000000   1.0000000            NaN      0.8509804
## Class: SIRA       0.0000000   0.9981763     0.00000000      0.8060874
##                  Precision    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.40102828 0.7878788 0.5315162 0.09705882     0.07647059
## Class: BOMBAY           NA 0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.09090909 0.4417178 0.1507853 0.11985294     0.05294118
## Class: DERMASON         NA 0.0000000        NA 0.26053922     0.00000000
## Class: HOROZ    0.60869565 0.9688581 0.7476636 0.14166667     0.13725490
## Class: SEKER            NA 0.0000000        NA 0.14901961     0.00000000
## Class: SIRA     0.00000000 0.0000000       NaN 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.190686275         0.8306929
## Class: BOMBAY            0.000000000         0.5000000
## Class: CALI              0.582352941         0.4201070
## Class: DERMASON          0.000000000         0.5000000
## Class: HOROZ             0.225490196         0.9330299
## Class: SEKER             0.000000000         0.5000000
## Class: SIRA              0.001470588         0.4990881
db_tda_pc_5.50.5_n3_db_nn1_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n3_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold<-(db_nn1_fit_re - db_tda_pc_5.50.5_n3_nn1_fit_re)
diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold
##     Accuracy
## 1 -0.2210061
## 2 -0.2935863
## 3 -0.2467718
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold
## $winLeft
## [1] 0.9909333
## 
## $winRope
## [1] 0.009066667
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n3_3_fold
## $left
## [1] 0.9950133
## 
## $rope
## [1] 0.0007182117
## 
## $right
## [1] 0.004268467
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold))
#bf_tda_pca_5.50.5_nn1.n3_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n3_3_fold)
## t = -11.946, df = 2, p-value = 0.006934
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.3451926 -0.1623836
## sample estimates:
##  mean of x 
## -0.2537881
### Test set diff
diff_drybean_tda_pca_5.50.5_nn1.n3_test<-(db_nn1_cf_ov_acc - db_tda_pc_5.50.5_n3_db_nn1_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_nn1.n3_test
##  Accuracy 
## 0.3431373
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n3_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n3_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n3_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n3_test$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n3_test$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n3_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n3_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n3_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n3_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1569667
## 
## $winRight
## [1] 0.8430333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n3_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n3_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n3_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nn1.n3_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n3_test)) #bf_tda_pca_5.50.5_nn1.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n3_test))


##Node4

#Neural Network 1
DryBean_TDA_PC_5.50.5_n4_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n4.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  46
## initial  value 1423.757504 
## iter  10 value 1322.547303
## iter  20 value 1179.442279
## iter  30 value 953.664190
## iter  40 value 787.809077
## iter  50 value 748.254529
## iter  60 value 734.887202
## iter  70 value 723.017604
## iter  80 value 714.355382
## iter  90 value 712.940930
## iter 100 value 650.937639
## final  value 650.937639 
## stopped after 100 iterations
## # weights:  67
## initial  value 1390.641654 
## iter  10 value 1324.492368
## iter  20 value 871.764487
## iter  30 value 728.993154
## iter  40 value 724.025447
## iter  50 value 721.612294
## iter  60 value 662.305491
## iter  70 value 486.411475
## iter  80 value 407.993551
## iter  90 value 399.988362
## iter 100 value 375.549522
## final  value 375.549522 
## stopped after 100 iterations
## # weights:  109
## initial  value 2539.871332 
## iter  10 value 1324.127145
## iter  20 value 1072.120855
## iter  30 value 1050.418264
## iter  40 value 1049.691710
## iter  50 value 1047.865388
## iter  60 value 830.209056
## iter  70 value 660.299531
## iter  80 value 613.390646
## iter  90 value 599.172606
## iter 100 value 502.855616
## final  value 502.855616 
## stopped after 100 iterations
## # weights:  151
## initial  value 1455.427716 
## iter  10 value 1367.787514
## iter  20 value 1302.021602
## iter  30 value 1047.299473
## iter  40 value 751.555281
## iter  50 value 738.666842
## iter  60 value 632.837461
## iter  70 value 603.266246
## iter  80 value 564.864108
## iter  90 value 437.425772
## iter 100 value 397.424099
## final  value 397.424099 
## stopped after 100 iterations
## # weights:  46
## initial  value 1716.615804 
## iter  10 value 1330.691859
## iter  20 value 1324.978188
## iter  30 value 1324.901137
## iter  40 value 1291.558979
## iter  50 value 1097.602603
## iter  60 value 1039.078607
## iter  70 value 1020.443959
## iter  80 value 890.057397
## iter  90 value 706.729294
## iter 100 value 667.375865
## final  value 667.375865 
## stopped after 100 iterations
## # weights:  67
## initial  value 2442.365932 
## iter  10 value 1325.411565
## iter  20 value 1324.224202
## iter  30 value 1323.362062
## iter  40 value 1268.807116
## iter  50 value 1141.892172
## iter  60 value 1122.903912
## iter  70 value 920.875954
## iter  80 value 797.573150
## iter  90 value 754.961329
## iter 100 value 746.981560
## final  value 746.981560 
## stopped after 100 iterations
## # weights:  109
## initial  value 1423.159828 
## iter  10 value 1325.941609
## iter  20 value 1324.295618
## iter  30 value 1324.276786
## iter  40 value 1258.632223
## iter  50 value 1053.932840
## iter  60 value 1035.469108
## iter  70 value 889.854150
## iter  80 value 758.722848
## iter  90 value 714.341905
## iter 100 value 709.240407
## final  value 709.240407 
## stopped after 100 iterations
## # weights:  151
## initial  value 2291.876913 
## iter  10 value 1336.092839
## iter  20 value 1318.651428
## iter  30 value 913.541920
## iter  40 value 819.812832
## iter  50 value 732.139665
## iter  60 value 669.403275
## iter  70 value 610.860651
## iter  80 value 592.242694
## iter  90 value 584.361383
## iter 100 value 556.316027
## final  value 556.316027 
## stopped after 100 iterations
## # weights:  46
## initial  value 1486.735337 
## iter  10 value 1324.138373
## iter  20 value 977.506962
## iter  30 value 806.411289
## iter  40 value 785.558889
## iter  50 value 742.788564
## iter  60 value 741.256528
## iter  70 value 737.036355
## iter  80 value 731.733705
## iter  90 value 721.124192
## iter 100 value 475.363327
## final  value 475.363327 
## stopped after 100 iterations
## # weights:  67
## initial  value 2332.645132 
## iter  10 value 1324.385061
## iter  10 value 1324.385054
## iter  10 value 1324.385049
## final  value 1324.385049 
## converged
## # weights:  109
## initial  value 1544.065159 
## iter  10 value 1323.453242
## iter  20 value 1283.588166
## iter  30 value 1027.312682
## iter  40 value 899.995788
## iter  50 value 750.101451
## iter  60 value 744.722849
## iter  70 value 738.861463
## iter  80 value 736.084340
## iter  90 value 732.342903
## iter 100 value 702.524040
## final  value 702.524040 
## stopped after 100 iterations
## # weights:  151
## initial  value 1763.316660 
## iter  10 value 1434.937009
## iter  20 value 1349.484937
## iter  30 value 1309.754645
## iter  40 value 1175.079959
## iter  50 value 983.618472
## iter  60 value 807.875823
## iter  70 value 782.519808
## iter  80 value 742.783321
## iter  90 value 726.077165
## iter 100 value 657.852375
## final  value 657.852375 
## stopped after 100 iterations
## # weights:  46
## initial  value 1465.634924 
## iter  10 value 1323.308717
## iter  20 value 1248.128529
## iter  30 value 1096.906526
## iter  40 value 807.051673
## iter  50 value 737.689771
## iter  60 value 720.334165
## iter  70 value 716.406460
## iter  80 value 715.464178
## iter  90 value 510.040783
## iter 100 value 457.832834
## final  value 457.832834 
## stopped after 100 iterations
## # weights:  67
## initial  value 1848.445548 
## iter  10 value 1323.107241
## iter  20 value 1323.065819
## iter  30 value 1316.070968
## iter  40 value 1144.148111
## iter  50 value 1012.035411
## iter  60 value 909.872654
## iter  70 value 870.205213
## iter  80 value 861.952095
## iter  90 value 858.405523
## iter 100 value 841.795464
## final  value 841.795464 
## stopped after 100 iterations
## # weights:  109
## initial  value 1395.510521 
## iter  10 value 1322.844196
## iter  20 value 1048.810882
## iter  30 value 836.567284
## iter  40 value 776.743347
## iter  50 value 716.165948
## iter  60 value 693.278488
## iter  70 value 690.506627
## iter  80 value 687.508114
## iter  90 value 681.330098
## iter 100 value 527.397181
## final  value 527.397181 
## stopped after 100 iterations
## # weights:  151
## initial  value 2145.077483 
## iter  10 value 1325.396461
## iter  20 value 894.530230
## iter  30 value 825.155437
## iter  40 value 794.984749
## iter  50 value 780.645177
## iter  60 value 733.565213
## iter  70 value 657.081025
## iter  80 value 557.695656
## iter  90 value 532.691878
## iter 100 value 525.028669
## final  value 525.028669 
## stopped after 100 iterations
## # weights:  46
## initial  value 1432.443096 
## iter  10 value 1325.569383
## iter  20 value 1323.259914
## iter  30 value 1323.228229
## final  value 1323.227858 
## converged
## # weights:  67
## initial  value 1793.871031 
## iter  10 value 1324.792193
## iter  20 value 1323.248712
## iter  30 value 1323.228096
## iter  40 value 1289.715011
## iter  50 value 809.114463
## iter  60 value 759.647414
## iter  70 value 677.037385
## iter  80 value 588.608761
## iter  90 value 566.850157
## iter 100 value 558.771402
## final  value 558.771402 
## stopped after 100 iterations
## # weights:  109
## initial  value 1593.185912 
## iter  10 value 1327.654472
## iter  20 value 1322.836191
## iter  30 value 1279.675752
## iter  40 value 1168.296394
## iter  50 value 899.917035
## iter  60 value 861.805154
## iter  70 value 854.707854
## iter  80 value 797.389458
## iter  90 value 755.546698
## iter 100 value 699.209953
## final  value 699.209953 
## stopped after 100 iterations
## # weights:  151
## initial  value 2036.054019 
## iter  10 value 1329.999693
## iter  20 value 1323.127998
## iter  30 value 1322.203527
## iter  40 value 1225.678088
## iter  50 value 1129.874598
## iter  60 value 903.769688
## iter  70 value 743.452940
## iter  80 value 727.836355
## iter  90 value 722.905487
## iter 100 value 722.488203
## final  value 722.488203 
## stopped after 100 iterations
## # weights:  46
## initial  value 1523.222362 
## iter  10 value 1325.616422
## iter  20 value 1323.961527
## iter  30 value 1323.953504
## iter  30 value 1323.953493
## iter  40 value 1321.011637
## iter  50 value 1050.422460
## iter  60 value 938.888617
## iter  70 value 895.364925
## iter  80 value 865.160509
## iter  90 value 854.758175
## iter 100 value 845.934241
## final  value 845.934241 
## stopped after 100 iterations
## # weights:  67
## initial  value 1420.205341 
## iter  10 value 1323.733268
## iter  20 value 1298.050015
## iter  30 value 1126.852570
## iter  40 value 1104.191650
## iter  50 value 970.078126
## iter  60 value 856.776954
## iter  70 value 843.566358
## iter  80 value 761.892310
## iter  90 value 742.409610
## iter 100 value 737.722054
## final  value 737.722054 
## stopped after 100 iterations
## # weights:  109
## initial  value 1705.124314 
## iter  10 value 1321.750276
## iter  20 value 1084.077899
## iter  30 value 1071.447291
## iter  40 value 917.348001
## iter  50 value 818.563153
## iter  60 value 769.080308
## iter  70 value 758.393448
## iter  80 value 669.744702
## iter  90 value 652.156780
## iter 100 value 648.002289
## final  value 648.002289 
## stopped after 100 iterations
## # weights:  151
## initial  value 1721.853758 
## iter  10 value 1324.645121
## iter  20 value 1306.095781
## iter  30 value 1046.830827
## iter  40 value 810.693662
## iter  50 value 758.484166
## iter  60 value 742.368307
## iter  70 value 731.806172
## iter  80 value 714.704209
## iter  90 value 678.249652
## iter 100 value 527.536429
## final  value 527.536429 
## stopped after 100 iterations
## # weights:  46
## initial  value 2008.671029 
## iter  10 value 1321.836831
## iter  20 value 1305.142764
## iter  30 value 1171.122465
## iter  40 value 947.345564
## iter  50 value 756.131794
## iter  60 value 666.349809
## iter  70 value 487.839062
## iter  80 value 386.186885
## iter  90 value 335.069523
## iter 100 value 321.044118
## final  value 321.044118 
## stopped after 100 iterations
## # weights:  67
## initial  value 1629.784319 
## iter  10 value 1321.779828
## iter  20 value 1321.753483
## iter  30 value 1319.373598
## iter  40 value 1062.606443
## iter  50 value 888.588423
## iter  60 value 856.592518
## iter  70 value 767.326271
## iter  80 value 635.946450
## iter  90 value 454.706226
## iter 100 value 399.605601
## final  value 399.605601 
## stopped after 100 iterations
## # weights:  109
## initial  value 1824.408888 
## iter  10 value 1321.712439
## iter  20 value 1287.730012
## iter  30 value 1200.483029
## iter  40 value 1038.787762
## iter  50 value 748.628181
## iter  60 value 659.036516
## iter  70 value 633.143021
## iter  80 value 591.535245
## iter  90 value 476.640428
## iter 100 value 401.676973
## final  value 401.676973 
## stopped after 100 iterations
## # weights:  151
## initial  value 1601.231261 
## iter  10 value 1324.448390
## iter  20 value 1321.778933
## iter  30 value 1319.790812
## iter  40 value 1305.831757
## iter  50 value 1100.140531
## iter  60 value 780.404159
## iter  70 value 733.343902
## iter  80 value 607.322290
## iter  90 value 577.075140
## iter 100 value 563.349778
## final  value 563.349778 
## stopped after 100 iterations
## # weights:  46
## initial  value 1895.174456 
## iter  10 value 1322.581995
## iter  20 value 1322.003630
## iter  30 value 1321.996811
## iter  40 value 1298.529830
## iter  50 value 1019.477130
## iter  60 value 775.189155
## iter  70 value 731.556210
## iter  80 value 730.230373
## iter  90 value 699.570250
## iter 100 value 584.964075
## final  value 584.964075 
## stopped after 100 iterations
## # weights:  67
## initial  value 1431.025344 
## iter  10 value 1322.803446
## iter  20 value 1322.006539
## iter  30 value 1321.996845
## iter  40 value 1149.378901
## iter  50 value 1065.921644
## iter  60 value 1047.717399
## iter  70 value 922.031266
## iter  80 value 884.147592
## iter  90 value 871.705797
## iter 100 value 855.583292
## final  value 855.583292 
## stopped after 100 iterations
## # weights:  109
## initial  value 1471.566041 
## iter  10 value 1322.486084
## iter  20 value 1322.002375
## iter  30 value 1321.975675
## iter  40 value 1265.436157
## iter  50 value 814.952054
## iter  60 value 751.017923
## iter  70 value 727.346347
## iter  80 value 723.658935
## iter  90 value 722.556523
## iter 100 value 721.752502
## final  value 721.752502 
## stopped after 100 iterations
## # weights:  151
## initial  value 1823.370950 
## iter  10 value 1324.203147
## iter  20 value 1322.025449
## iter  30 value 1321.996470
## iter  40 value 1317.910914
## iter  50 value 1169.147310
## iter  60 value 1155.921445
## iter  70 value 866.675774
## iter  80 value 810.899488
## iter  90 value 744.491774
## iter 100 value 733.098765
## final  value 733.098765 
## stopped after 100 iterations
## # weights:  46
## initial  value 1418.739965 
## iter  10 value 1320.956817
## iter  20 value 1161.086619
## iter  30 value 1110.581933
## iter  40 value 892.544634
## iter  50 value 793.322794
## iter  60 value 751.833277
## iter  70 value 732.991426
## iter  80 value 696.287349
## iter  90 value 539.979137
## iter 100 value 360.631065
## final  value 360.631065 
## stopped after 100 iterations
## # weights:  67
## initial  value 2132.083238 
## iter  10 value 1322.774458
## iter  20 value 1322.766315
## iter  30 value 1322.733987
## iter  40 value 1322.726335
## iter  50 value 1321.971625
## iter  60 value 1319.445176
## iter  70 value 1302.761576
## iter  80 value 1300.850449
## iter  90 value 840.726686
## iter 100 value 780.362437
## final  value 780.362437 
## stopped after 100 iterations
## # weights:  109
## initial  value 2073.734671 
## iter  10 value 1324.841108
## iter  20 value 1285.710497
## iter  30 value 1164.607269
## iter  40 value 1077.633609
## iter  50 value 811.478788
## iter  60 value 745.971402
## iter  70 value 733.032855
## iter  80 value 730.540437
## iter  90 value 699.369829
## iter 100 value 591.500339
## final  value 591.500339 
## stopped after 100 iterations
## # weights:  151
## initial  value 1461.912456 
## iter  10 value 1321.829204
## iter  20 value 1129.371432
## iter  30 value 949.494409
## iter  40 value 867.667042
## iter  50 value 795.952913
## iter  60 value 776.571969
## iter  70 value 732.235832
## iter  80 value 728.406597
## iter  90 value 726.759742
## iter 100 value 695.629163
## final  value 695.629163 
## stopped after 100 iterations
## # weights:  109
## initial  value 2169.478651 
## iter  10 value 1983.975350
## iter  20 value 1827.227207
## iter  30 value 1222.218733
## iter  40 value 1125.029779
## iter  50 value 1063.266824
## iter  60 value 1054.155666
## iter  70 value 1014.031922
## iter  80 value 792.870596
## iter  90 value 667.532689
## iter 100 value 580.024116
## final  value 580.024116 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n4_NN1Fit0
## Neural Network 
## 
## 1590 samples
##   16 predictor
##    4 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'HOROZ' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1061, 1060, 1059 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   2     0.3    0.8463874  0.7592319
##   2     0.5    0.6150948  0.3559886
##   2     0.7    0.8125665  0.7072849
##   3     0.3    0.8094242  0.7004414
##   3     0.5    0.6994249  0.5216280
##   3     0.7    0.6123972  0.3521753
##   5     0.3    0.8616064  0.7940362
##   5     0.5    0.7295603  0.5690515
##   5     0.7    0.7829199  0.6606979
##   7     0.3    0.8202108  0.7220827
##   7     0.5    0.7321176  0.5706245
##   7     0.7    0.7547139  0.6125326
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 5 and decay = 0.3.
DryBean_TDA_PC_5.50.5_n4_NN1Fit0$resample
##    Accuracy     Kappa Resample
## 1 0.8487713 0.7734099    Fold1
## 2 0.8945386 0.8425234    Fold3
## 3 0.8415094 0.7661752    Fold2
db_tda_pc_5.50.5_n4_nn1_fit_re<-DryBean_TDA_PC_5.50.5_n4_NN1Fit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n4_NN1Fit0)
## a 16-5-4 network with 109 weights
## options were - softmax modelling  decay=0.3
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00   -0.15   -3.08    0.92   -0.48    0.01    0.00    0.18    0.10    0.01 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.00    0.01    0.00    0.00    0.00    0.00    0.00    0.01    0.00    0.00 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##    0.00    0.02    0.00    0.00    0.00    0.00    0.00    0.02    0.00    0.00 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h4  i1->h4  i2->h4  i3->h4  i4->h4  i5->h4  i6->h4  i7->h4  i8->h4  i9->h4 
##    0.03    0.01    1.93   -0.36    0.18    0.12    0.02   -0.03   -1.81   -0.33 
## i10->h4 i11->h4 i12->h4 i13->h4 i14->h4 i15->h4 i16->h4 
##    0.02   -0.01    0.01    0.00    0.00    0.01    0.02 
##   b->h5  i1->h5  i2->h5  i3->h5  i4->h5  i5->h5  i6->h5  i7->h5  i8->h5  i9->h5 
##    0.00   -0.02    0.00    0.00    0.00    0.00    0.00   -0.02    0.00    0.00 
## i10->h5 i11->h5 i12->h5 i13->h5 i14->h5 i15->h5 i16->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##  b->o1 h1->o1 h2->o1 h3->o1 h4->o1 h5->o1 
##   0.23  -2.03  -0.04   0.23   0.23   0.00 
##  b->o2 h1->o2 h2->o2 h3->o2 h4->o2 h5->o2 
##  -0.92   5.86   0.09  -0.94  -1.09   0.00 
##  b->o3 h1->o3 h2->o3 h3->o3 h4->o3 h5->o3 
##   1.23  -2.59   0.02   1.24  -2.93   0.00 
##  b->o4 h1->o4 h2->o4 h3->o4 h4->o4 h5->o4 
##  -0.54  -1.23  -0.07  -0.53   3.79   0.00
#vip(DryBean_TDA_PC_5.50.5_n4_NN1Fit0,25) + ggtitle("dryBean_TDA_PCA_5.50.5_n4_NN1Fit TDA-Assited NN")

# Predict outcome using DryBean_TDA_PC_5.50.5_n4_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n4_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n4_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n4_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0    156    0        0     0     0    0
##   CALI          145      0  418        0     7     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ         251      0   71     1063   571   608  790
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.2806          
##                  95% CI : (0.2669, 0.2947)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 0.001947        
##                                           
##                   Kappa : 0.1687          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000      0.8548          0.0000
## Specificity                  1.00000       1.00000      0.9577          1.0000
## Pos Pred Value                   NaN       1.00000      0.7333             NaN
## Neg Pred Value               0.90294       1.00000      0.9798          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.03824      0.1025          0.0000
## Detection Prevalence         0.00000       0.03824      0.1397          0.0000
## Balanced Accuracy            0.50000       1.00000      0.9062          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9879        0.000      0.0000
## Specificity                0.2053        1.000      1.0000
## Pos Pred Value             0.1702          NaN         NaN
## Neg Pred Value             0.9904        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1400        0.000      0.0000
## Detection Prevalence       0.8221        0.000      0.0000
## Balanced Accuracy          0.5966        0.500      0.5000
db_tda_pc_5.50.5_n4_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0    156    0        0     0     0    0
##   CALI          145      0  418        0     7     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ         251      0   71     1063   571   608  790
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.2806          
##                  95% CI : (0.2669, 0.2947)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 0.001947        
##                                           
##                   Kappa : 0.1687          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000      0.8548          0.0000
## Specificity                  1.00000       1.00000      0.9577          1.0000
## Pos Pred Value                   NaN       1.00000      0.7333             NaN
## Neg Pred Value               0.90294       1.00000      0.9798          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.03824      0.1025          0.0000
## Detection Prevalence         0.00000       0.03824      0.1397          0.0000
## Balanced Accuracy            0.50000       1.00000      0.9062          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9879        0.000      0.0000
## Specificity                0.2053        1.000      1.0000
## Pos Pred Value             0.1702          NaN         NaN
## Neg Pred Value             0.9904        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1400        0.000      0.0000
## Detection Prevalence       0.8221        0.000      0.0000
## Balanced Accuracy          0.5966        0.500      0.5000
db_tda_pc_5.50.5_n4_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##     0.28063725     0.16868927     0.26688891     0.29470037     0.26053922 
## AccuracyPValue  McnemarPValue 
##     0.00194719            NaN
db_tda_pc_5.50.5_n4_db_nn1_cf0_ov_acc<-db_tda_pc_5.50.5_n4_db_nn1_cf0$overall[1]
db_tda_pc_5.50.5_n4_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.0000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY     1.0000000   1.0000000      1.0000000      1.0000000 1.0000000
## Class: CALI       0.8548057   0.9576720      0.7333333      0.9797721 0.7333333
## Class: DERMASON   0.0000000   1.0000000            NaN      0.7394608        NA
## Class: HOROZ      0.9878893   0.2053113      0.1702445      0.9903581 0.1702445
## Class: SEKER      0.0000000   1.0000000            NaN      0.8509804        NA
## Class: SIRA       0.0000000   1.0000000            NaN      0.8063725        NA
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.0000000        NA 0.09705882     0.00000000
## Class: BOMBAY   1.0000000 1.0000000 0.03823529     0.03823529
## Class: CALI     0.8548057 0.7894240 0.11985294     0.10245098
## Class: DERMASON 0.0000000        NA 0.26053922     0.00000000
## Class: HOROZ    0.9878893 0.2904374 0.14166667     0.13995098
## Class: SEKER    0.0000000        NA 0.14901961     0.00000000
## Class: SIRA     0.0000000        NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.00000000         0.5000000
## Class: BOMBAY             0.03823529         1.0000000
## Class: CALI               0.13970588         0.9062388
## Class: DERMASON           0.00000000         0.5000000
## Class: HOROZ              0.82205882         0.5966003
## Class: SEKER              0.00000000         0.5000000
## Class: SIRA               0.00000000         0.5000000
db_tda_pc_5.50.5_n4_db_nn1_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n4_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold<-(db_nn1_fit_re - db_tda_pc_5.50.5_n4_nn1_fit_re)
diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold
##     Accuracy
## 1 -0.1431667
## 2 -0.3613312
## 3 -0.2741715
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold
## $winLeft
## [1] 0.9909333
## 
## $winRope
## [1] 0.009066667
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n4_3_fold
## $left
## [1] 0.9618324
## 
## $rope
## [1] 0.004922686
## 
## $right
## [1] 0.03324494
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold))
#bf_tda_pca_5.50.5_nn1.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n4_3_fold)
## t = -4.0939, df = 2, p-value = 0.05481
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.53234974  0.01323681
## sample estimates:
##  mean of x 
## -0.2595565
### Test set diff
diff_drybean_tda_pca_5.50.5_nn1.n4_test<-(db_nn1_cf_ov_acc - db_tda_pc_5.50.5_n4_db_nn1_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_nn1.n4_test
##  Accuracy 
## 0.3291667
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n4_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n4_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n4_test$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n4_test$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n4_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.164
## 
## $winRight
## [1] 0.836
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n4_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nn1.n4_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n4_test)) #bf_tda_pca_5.50.5_nn1.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n4_test))


##Node5

#Neural Network 1
DryBean_TDA_PC_5.50.5_n5_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n5.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  37
## initial  value 330.975925 
## iter  10 value 10.529011
## iter  20 value 10.441081
## iter  30 value 7.781758
## iter  40 value 7.178670
## iter  50 value 6.512430
## iter  60 value 5.770999
## iter  70 value 5.766826
## iter  80 value 5.765927
## iter  90 value 5.765810
## final  value 5.765790 
## converged
## # weights:  55
## initial  value 155.463032 
## iter  10 value 12.076439
## iter  20 value 9.362283
## iter  30 value 9.362218
## iter  40 value 9.251706
## iter  50 value 8.639629
## iter  60 value 6.620488
## iter  70 value 5.299559
## iter  80 value 5.079599
## iter  90 value 4.977372
## iter 100 value 4.934320
## final  value 4.934320 
## stopped after 100 iterations
## # weights:  91
## initial  value 250.386966 
## iter  10 value 8.268968
## iter  20 value 7.077836
## iter  30 value 6.067821
## iter  40 value 5.850982
## iter  50 value 5.324141
## iter  60 value 4.242711
## iter  70 value 3.995921
## iter  80 value 3.966817
## iter  90 value 3.964395
## iter 100 value 3.909306
## final  value 3.909306 
## stopped after 100 iterations
## # weights:  127
## initial  value 384.495175 
## iter  10 value 11.395886
## iter  20 value 8.083242
## iter  30 value 7.715808
## iter  40 value 7.118165
## iter  50 value 5.963560
## iter  60 value 5.324041
## iter  70 value 5.053133
## iter  80 value 4.704147
## iter  90 value 4.474520
## iter 100 value 4.023932
## final  value 4.023932 
## stopped after 100 iterations
## # weights:  37
## initial  value 429.463839 
## iter  10 value 17.050565
## iter  20 value 11.064412
## iter  30 value 10.898644
## iter  40 value 10.898075
## iter  50 value 10.895718
## iter  60 value 10.719052
## iter  70 value 8.859840
## iter  80 value 7.676699
## iter  90 value 7.660207
## iter 100 value 7.656439
## final  value 7.656439 
## stopped after 100 iterations
## # weights:  55
## initial  value 232.077831 
## iter  10 value 11.735423
## iter  20 value 11.129678
## iter  30 value 10.263033
## iter  40 value 9.177586
## iter  50 value 8.496739
## iter  60 value 8.131594
## iter  70 value 7.650441
## iter  80 value 7.436428
## iter  90 value 7.309205
## iter 100 value 6.975349
## final  value 6.975349 
## stopped after 100 iterations
## # weights:  91
## initial  value 701.316011 
## iter  10 value 17.664352
## iter  20 value 17.586434
## iter  30 value 11.850141
## iter  40 value 8.951678
## iter  50 value 8.899924
## iter  60 value 8.809852
## iter  70 value 8.376801
## iter  80 value 7.731936
## iter  90 value 7.023255
## iter 100 value 6.126855
## final  value 6.126855 
## stopped after 100 iterations
## # weights:  127
## initial  value 334.098119 
## iter  10 value 22.987765
## iter  20 value 21.925718
## iter  30 value 21.184238
## iter  40 value 9.959539
## iter  50 value 9.167946
## iter  60 value 8.838899
## iter  70 value 7.951030
## iter  80 value 7.547414
## iter  90 value 6.876657
## iter 100 value 6.601309
## final  value 6.601309 
## stopped after 100 iterations
## # weights:  37
## initial  value 318.872962 
## iter  10 value 14.543180
## iter  20 value 14.450271
## iter  30 value 13.162107
## iter  40 value 12.378518
## iter  50 value 12.338011
## iter  60 value 12.329102
## iter  70 value 12.324819
## iter  80 value 12.294576
## iter  90 value 12.218869
## iter 100 value 11.258269
## final  value 11.258269 
## stopped after 100 iterations
## # weights:  55
## initial  value 276.282907 
## iter  10 value 13.097704
## iter  20 value 11.081413
## iter  30 value 10.859967
## iter  40 value 10.444283
## iter  50 value 7.773453
## iter  60 value 7.610163
## iter  70 value 7.600651
## iter  80 value 7.596402
## iter  90 value 7.595145
## iter 100 value 7.594995
## final  value 7.594995 
## stopped after 100 iterations
## # weights:  91
## initial  value 182.469028 
## iter  10 value 14.606902
## iter  20 value 14.508895
## iter  30 value 14.419357
## iter  40 value 12.287494
## iter  50 value 10.160711
## iter  60 value 9.320988
## iter  70 value 8.581432
## iter  80 value 8.311145
## iter  90 value 7.137264
## iter 100 value 6.720092
## final  value 6.720092 
## stopped after 100 iterations
## # weights:  127
## initial  value 279.111549 
## iter  10 value 11.774832
## iter  20 value 10.998805
## iter  30 value 9.910084
## iter  40 value 9.285195
## iter  50 value 9.174113
## iter  60 value 8.610866
## iter  70 value 7.773089
## iter  80 value 7.432637
## iter  90 value 7.137539
## iter 100 value 6.837281
## final  value 6.837281 
## stopped after 100 iterations
## # weights:  37
## initial  value 301.641103 
## iter  10 value 9.374783
## iter  20 value 9.369438
## iter  30 value 9.353473
## iter  40 value 9.244437
## iter  50 value 9.102951
## iter  60 value 7.273730
## iter  70 value 7.238786
## iter  80 value 7.229217
## iter  90 value 7.228270
## iter 100 value 7.228027
## final  value 7.228027 
## stopped after 100 iterations
## # weights:  55
## initial  value 239.589545 
## iter  10 value 13.604353
## iter  20 value 9.227744
## iter  30 value 8.750709
## iter  40 value 8.612530
## iter  50 value 7.926781
## iter  60 value 6.742459
## iter  70 value 5.532885
## iter  80 value 5.294815
## iter  90 value 4.696065
## iter 100 value 4.675240
## final  value 4.675240 
## stopped after 100 iterations
## # weights:  91
## initial  value 62.791463 
## iter  10 value 13.213617
## iter  20 value 8.298140
## iter  30 value 7.396596
## iter  40 value 7.122372
## iter  50 value 6.084309
## iter  60 value 5.514858
## iter  70 value 5.229513
## iter  80 value 4.956976
## iter  90 value 4.736555
## iter 100 value 4.338883
## final  value 4.338883 
## stopped after 100 iterations
## # weights:  127
## initial  value 216.188245 
## iter  10 value 7.892441
## iter  20 value 7.891364
## iter  30 value 7.371293
## iter  40 value 7.205767
## iter  50 value 6.407298
## iter  60 value 5.280510
## iter  70 value 4.918686
## iter  80 value 4.567606
## iter  90 value 4.338019
## iter 100 value 4.025482
## final  value 4.025482 
## stopped after 100 iterations
## # weights:  37
## initial  value 148.912336 
## iter  10 value 13.545524
## iter  20 value 12.141697
## iter  30 value 9.254637
## iter  40 value 9.110623
## iter  50 value 8.385853
## iter  60 value 7.683702
## iter  70 value 7.667398
## iter  80 value 7.663390
## iter  90 value 7.662521
## iter 100 value 7.662050
## final  value 7.662050 
## stopped after 100 iterations
## # weights:  55
## initial  value 295.527735 
## iter  10 value 12.616680
## iter  20 value 11.160131
## iter  30 value 8.494414
## iter  40 value 8.377531
## iter  50 value 7.843031
## iter  60 value 7.365818
## iter  70 value 7.041623
## iter  80 value 6.288786
## iter  90 value 6.261710
## iter 100 value 6.256490
## final  value 6.256490 
## stopped after 100 iterations
## # weights:  91
## initial  value 166.601931 
## iter  10 value 21.855608
## iter  20 value 10.702841
## iter  30 value 8.966772
## iter  40 value 8.743832
## iter  50 value 8.244048
## iter  60 value 7.574714
## iter  70 value 7.238070
## iter  80 value 6.712253
## iter  90 value 5.711994
## iter 100 value 5.494025
## final  value 5.494025 
## stopped after 100 iterations
## # weights:  127
## initial  value 110.166876 
## iter  10 value 9.285605
## iter  20 value 8.974528
## iter  30 value 8.956882
## iter  40 value 8.956627
## iter  50 value 8.800857
## iter  60 value 8.264602
## iter  70 value 7.321806
## iter  80 value 6.593819
## iter  90 value 6.177986
## iter 100 value 6.053944
## final  value 6.053944 
## stopped after 100 iterations
## # weights:  37
## initial  value 253.202703 
## iter  10 value 16.011492
## iter  20 value 12.777227
## iter  30 value 11.289253
## iter  40 value 10.671793
## iter  50 value 10.305795
## iter  60 value 9.332956
## iter  70 value 9.282944
## iter  80 value 9.280375
## iter  90 value 9.280054
## iter 100 value 9.279904
## final  value 9.279904 
## stopped after 100 iterations
## # weights:  55
## initial  value 193.411083 
## iter  10 value 13.122406
## iter  20 value 12.353991
## iter  30 value 11.770252
## iter  40 value 9.380961
## iter  50 value 8.557710
## iter  60 value 7.647395
## iter  70 value 7.607068
## iter  80 value 7.603644
## iter  90 value 7.602615
## iter 100 value 7.602102
## final  value 7.602102 
## stopped after 100 iterations
## # weights:  91
## initial  value 180.811750 
## iter  10 value 10.757291
## iter  20 value 10.079706
## iter  30 value 9.615206
## iter  40 value 9.444724
## iter  50 value 9.047579
## iter  60 value 8.578090
## iter  70 value 8.090973
## iter  80 value 7.688514
## iter  90 value 7.204510
## iter 100 value 6.601469
## final  value 6.601469 
## stopped after 100 iterations
## # weights:  127
## initial  value 370.586297 
## iter  10 value 12.636623
## iter  20 value 12.320300
## iter  30 value 11.963664
## iter  40 value 11.827853
## iter  50 value 10.719888
## iter  60 value 9.279646
## iter  70 value 9.070265
## iter  80 value 8.808843
## iter  90 value 8.436711
## iter 100 value 7.173954
## final  value 7.173954 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  37
## initial  value 289.803717 
## iter  10 value 5.119959
## iter  20 value 4.154443
## final  value 4.154443 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  55
## initial  value 276.223193 
## iter  10 value 5.708488
## iter  20 value 3.477317
## iter  30 value 3.374424
## iter  40 value 3.359989
## iter  50 value 3.359788
## final  value 3.359788 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  91
## initial  value 174.346234 
## iter  10 value 7.445037
## iter  20 value 3.827063
## iter  30 value 2.481636
## iter  40 value 2.481044
## iter  50 value 2.481011
## final  value 2.481010 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  127
## initial  value 154.889377 
## iter  10 value 6.642537
## iter  20 value 3.579117
## iter  30 value 2.213705
## iter  40 value 2.212138
## iter  50 value 2.209716
## iter  60 value 2.073360
## iter  70 value 1.996633
## iter  80 value 1.996285
## iter  90 value 1.995785
## iter 100 value 1.995481
## final  value 1.995481 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  37
## initial  value 323.520685 
## iter  10 value 7.049902
## iter  20 value 6.018559
## final  value 6.018557 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  55
## initial  value 168.140057 
## iter  10 value 10.136123
## iter  20 value 6.317699
## iter  30 value 4.985054
## iter  40 value 4.892131
## iter  50 value 4.891047
## iter  50 value 4.891047
## iter  60 value 4.889698
## final  value 4.889678 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  91
## initial  value 87.905872 
## iter  10 value 15.339173
## iter  20 value 6.115757
## iter  30 value 4.596207
## iter  40 value 4.169809
## iter  50 value 3.709675
## iter  60 value 3.633435
## iter  70 value 3.632513
## iter  80 value 3.632504
## iter  90 value 3.632492
## iter 100 value 3.632471
## final  value 3.632471 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  127
## initial  value 444.964425 
## iter  10 value 14.832112
## iter  20 value 4.725693
## iter  30 value 4.268668
## iter  40 value 3.963392
## iter  50 value 3.717371
## iter  60 value 3.356497
## iter  70 value 3.323176
## iter  80 value 3.313583
## iter  90 value 3.241582
## iter 100 value 3.241309
## final  value 3.241309 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  37
## initial  value 169.208781 
## iter  10 value 8.584812
## iter  20 value 7.646229
## final  value 7.646229 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  55
## initial  value 189.898610 
## iter  10 value 16.198008
## iter  20 value 6.317823
## iter  30 value 6.233678
## iter  40 value 6.232667
## iter  50 value 6.232655
## iter  50 value 6.232655
## iter  50 value 6.232655
## final  value 6.232655 
## converged
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  91
## initial  value 197.680995 
## iter  10 value 7.684905
## iter  20 value 6.283613
## iter  30 value 5.868712
## iter  40 value 5.645912
## iter  50 value 5.318940
## iter  60 value 5.308764
## iter  70 value 5.308563
## iter  80 value 5.308388
## iter  90 value 5.308345
## iter 100 value 4.662843
## final  value 4.662843 
## stopped after 100 iterations
## Warning in nnet.formula(.outcome ~ ., data = dat, size = param$size, decay =
## param$decay, : group 'CALI' is empty
## # weights:  127
## initial  value 385.158316 
## iter  10 value 8.568458
## iter  20 value 7.666189
## iter  30 value 7.012669
## iter  40 value 4.785432
## iter  50 value 4.664175
## iter  60 value 4.662060
## iter  70 value 4.167561
## iter  80 value 4.154567
## iter  90 value 3.792074
## iter 100 value 3.770756
## final  value 3.770756 
## stopped after 100 iterations
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo,
## : There were missing values in resampled performance measures.
## # weights:  37
## initial  value 429.970293 
## iter  10 value 13.590228
## iter  20 value 13.518697
## iter  30 value 13.086444
## iter  40 value 10.684221
## iter  50 value 10.327693
## iter  60 value 10.314934
## iter  70 value 10.311224
## iter  80 value 10.309263
## iter  90 value 10.308737
## iter 100 value 10.308436
## final  value 10.308436 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n5_NN1Fit0
## Neural Network 
## 
## 417 samples
##  16 predictor
##   2 classes: 'BOMBAY', 'CALI' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 278, 279, 277 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy  Kappa
##   2     0.3    0.997619  0    
##   2     0.5    0.997619  0    
##   2     0.7    0.997619  0    
##   3     0.3    0.997619  0    
##   3     0.5    0.997619  0    
##   3     0.7    0.997619  0    
##   5     0.3    0.997619  0    
##   5     0.5    0.997619  0    
##   5     0.7    0.997619  0    
##   7     0.3    0.997619  0    
##   7     0.5    0.997619  0    
##   7     0.7    0.997619  0    
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 2 and decay = 0.7.
DryBean_TDA_PC_5.50.5_n5_NN1Fit0$resample
##    Accuracy Kappa Resample
## 1 0.9928571     0    Fold3
## 2 1.0000000    NA    Fold2
## 3 1.0000000    NA    Fold1
db_tda_pc_5.50.5_n5_nn1_fit_re<-DryBean_TDA_PC_5.50.5_n5_NN1Fit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n5_NN1Fit0)
## a 16-2-1 network with 37 weights
## options were - entropy fitting  decay=0.7
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00    0.00   -0.05   -0.02   -0.01    0.00    0.00    0.00   -0.01    0.00 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.00    0.00   -0.04   -0.02    0.00    0.00    0.00    0.00   -0.01    0.00 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##  b->o h1->o h2->o 
## -1.31 -1.87 -1.87
#vip(DryBean_TDA_PC_5.50.5_n5_NN1Fit0,25) + ggtitle("dryBean_TDA_PCA_5.50.5_n5_NN1Fit TDA-Assited NN")

# Predict outcome using DryBean_TDA_PC_5.50.5_n5_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n5_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n5_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n5_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY        396    156  489     1063   578   608  790
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.0382          
##                  95% CI : (0.0326, 0.0446)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000      0.0000          0.0000
## Specificity                  1.00000       0.00000      1.0000          1.0000
## Pos Pred Value                   NaN       0.03824         NaN             NaN
## Neg Pred Value               0.90294           NaN      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.03824      0.0000          0.0000
## Detection Prevalence         0.00000       1.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n5_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY        396    156  489     1063   578   608  790
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.0382          
##                  95% CI : (0.0326, 0.0446)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000      0.0000          0.0000
## Specificity                  1.00000       0.00000      1.0000          1.0000
## Pos Pred Value                   NaN       0.03824         NaN             NaN
## Neg Pred Value               0.90294           NaN      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.03824      0.0000          0.0000
## Detection Prevalence         0.00000       1.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n5_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##     0.03823529     0.00000000     0.03256139     0.04458199     0.26053922 
## AccuracyPValue  McnemarPValue 
##     1.00000000            NaN
db_tda_pc_5.50.5_n5_db_nn1_cf0_ov_acc<-db_tda_pc_5.50.5_n5_db_nn1_cf0$overall[1]
db_tda_pc_5.50.5_n5_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value
## Class: BARBUNYA           0           1            NaN      0.9029412
## Class: BOMBAY             1           0     0.03823529            NaN
## Class: CALI               0           1            NaN      0.8801471
## Class: DERMASON           0           1            NaN      0.7394608
## Class: HOROZ              0           1            NaN      0.8583333
## Class: SEKER              0           1            NaN      0.8509804
## Class: SIRA               0           1            NaN      0.8063725
##                  Precision Recall         F1 Prevalence Detection Rate
## Class: BARBUNYA         NA      0         NA 0.09705882     0.00000000
## Class: BOMBAY   0.03823529      1 0.07365439 0.03823529     0.03823529
## Class: CALI             NA      0         NA 0.11985294     0.00000000
## Class: DERMASON         NA      0         NA 0.26053922     0.00000000
## Class: HOROZ            NA      0         NA 0.14166667     0.00000000
## Class: SEKER            NA      0         NA 0.14901961     0.00000000
## Class: SIRA             NA      0         NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA                    0               0.5
## Class: BOMBAY                      1               0.5
## Class: CALI                        0               0.5
## Class: DERMASON                    0               0.5
## Class: HOROZ                       0               0.5
## Class: SEKER                       0               0.5
## Class: SIRA                        0               0.5
db_tda_pc_5.50.5_n5_db_nn1_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n5_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold<-(db_nn1_fit_re - db_tda_pc_5.50.5_n5_nn1_fit_re)
diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold
##     Accuracy
## 1 -0.2872526
## 2 -0.4667926
## 3 -0.4326621
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold
## $winLeft
## [1] 0.9926
## 
## $winRope
## [1] 0.0074
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n5_3_fold
## $left
## [1] 0.9869414
## 
## $rope
## [1] 0.001211897
## 
## $right
## [1] 0.01184669
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold))
#bf_tda_pca_5.50.5_nn1.n5_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_nn1_n5_3_fold)
## t = -7.186, df = 2, p-value = 0.01882
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.6324178 -0.1587203
## sample estimates:
##  mean of x 
## -0.3955691
### Test set diff
diff_drybean_tda_pca_5.50.5_nn1.n5_test<-(db_nn1_cf_ov_acc - db_tda_pc_5.50.5_n5_db_nn1_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_nn1.n5_test
##  Accuracy 
## 0.5715686
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nn1.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n5_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nn1.n5_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nn1.n5_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nn1.n5_test$probLeft/bst_dbf_db_tda_pca_5.50.5_nn1.n5_test$probRight
bst_dbf_db_tda_pca_5.50.5_nn1.n5_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nn1.n5_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n5_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nn1.n5_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1571
## 
## $winRight
## [1] 0.8429
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nn1.n5_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n5_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nn1.n5_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nn1.n5_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nn1.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n5_test)) #bf_tda_pca_5.50.5_nn1.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nn1.n5_test))


##With TDA KDE filter 5 intervals, 50% overlap, 5 bins 
##Node1

#Neural Network 1
DryBean_TDA_KDE_5.50.5_n1_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n1.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  55
## initial  value 11209.806029 
## iter  10 value 10567.853418
## final  value 10567.846799 
## converged
## # weights:  79
## initial  value 11131.014641 
## iter  10 value 10568.045002
## iter  20 value 8920.687141
## iter  30 value 8573.018814
## iter  40 value 8270.148421
## iter  50 value 8207.008452
## iter  60 value 8153.264403
## iter  70 value 8141.517883
## iter  80 value 8122.431755
## iter  90 value 8087.332916
## iter  90 value 8087.332851
## final  value 8087.332851 
## converged
## # weights:  127
## initial  value 12640.791580 
## iter  10 value 10567.786135
## final  value 10567.783340 
## converged
## # weights:  175
## initial  value 14611.007046 
## iter  10 value 10595.637989
## iter  20 value 10567.481620
## iter  30 value 10083.358534
## iter  40 value 9823.370128
## iter  50 value 9702.557302
## iter  60 value 9667.040259
## iter  70 value 9509.163732
## iter  80 value 9485.075703
## iter  90 value 9334.225638
## iter 100 value 9092.395212
## final  value 9092.395212 
## stopped after 100 iterations
## # weights:  55
## initial  value 12172.206983 
## iter  10 value 10650.573540
## iter  20 value 10568.293324
## iter  30 value 10567.974924
## final  value 10567.974689 
## converged
## # weights:  79
## initial  value 11980.627796 
## iter  10 value 10574.427271
## iter  20 value 10567.948846
## iter  30 value 10567.868920
## final  value 10567.868003 
## converged
## # weights:  127
## initial  value 13948.568881 
## iter  10 value 10819.707676
## iter  20 value 10569.285767
## iter  30 value 10567.986521
## iter  40 value 10567.970115
## iter  50 value 10567.875306
## final  value 10567.868771 
## converged
## # weights:  175
## initial  value 13260.639353 
## iter  10 value 10580.472350
## iter  20 value 10507.172682
## iter  30 value 8913.632053
## iter  40 value 8353.889697
## iter  50 value 7148.135508
## iter  60 value 6868.547859
## iter  70 value 6454.403530
## iter  80 value 6044.123453
## iter  90 value 6008.893709
## iter 100 value 5630.857530
## final  value 5630.857530 
## stopped after 100 iterations
## # weights:  55
## initial  value 11654.000970 
## iter  10 value 10661.257865
## iter  20 value 10568.844049
## iter  30 value 10568.094218
## iter  40 value 10557.148108
## iter  50 value 10547.694920
## iter  60 value 10543.303613
## iter  70 value 8710.024238
## iter  80 value 8502.756876
## iter  90 value 8211.167963
## iter 100 value 8083.407656
## final  value 8083.407656 
## stopped after 100 iterations
## # weights:  79
## initial  value 11922.894148 
## iter  10 value 10571.528636
## iter  20 value 10568.029735
## iter  30 value 10567.957676
## iter  40 value 10567.881791
## final  value 10567.878654 
## converged
## # weights:  127
## initial  value 11870.828835 
## iter  10 value 10568.757049
## final  value 10567.952799 
## converged
## # weights:  175
## initial  value 15388.204855 
## iter  10 value 10651.811677
## iter  20 value 10566.563677
## iter  30 value 8873.736453
## iter  40 value 8787.364631
## iter  50 value 8781.197554
## iter  60 value 8680.217715
## iter  70 value 8473.739483
## iter  80 value 7694.753808
## iter  90 value 6776.116031
## iter 100 value 4677.700397
## final  value 4677.700397 
## stopped after 100 iterations
## # weights:  55
## initial  value 12075.838161 
## iter  10 value 10565.444067
## final  value 10565.375333 
## converged
## # weights:  79
## initial  value 11862.031093 
## iter  10 value 10565.393822
## final  value 10565.375369 
## converged
## # weights:  127
## initial  value 10894.859108 
## iter  10 value 10565.391241
## iter  20 value 10554.166758
## iter  30 value 10503.399737
## iter  40 value 10397.334455
## iter  50 value 9813.014348
## iter  60 value 7427.971813
## iter  70 value 6385.771106
## iter  80 value 5929.112047
## iter  90 value 5546.698981
## iter 100 value 5445.329815
## final  value 5445.329815 
## stopped after 100 iterations
## # weights:  175
## initial  value 12395.444345 
## iter  10 value 10565.311257
## iter  20 value 9962.847495
## iter  30 value 9275.780373
## iter  40 value 9117.746996
## iter  50 value 8618.941392
## iter  60 value 8533.556531
## iter  70 value 8472.086923
## iter  80 value 7432.620596
## iter  90 value 7098.533064
## iter 100 value 7022.632561
## final  value 7022.632561 
## stopped after 100 iterations
## # weights:  55
## initial  value 11395.196601 
## iter  10 value 10585.744174
## iter  20 value 10565.754495
## iter  30 value 10564.163485
## iter  40 value 9561.925669
## iter  50 value 9407.709209
## iter  60 value 9396.132481
## iter  70 value 9387.124978
## iter  80 value 9365.894433
## iter  90 value 9333.421092
## iter 100 value 9329.387456
## final  value 9329.387456 
## stopped after 100 iterations
## # weights:  79
## initial  value 11917.780266 
## iter  10 value 10578.937778
## iter  20 value 10565.575169
## iter  30 value 10564.744871
## iter  40 value 10164.027798
## iter  50 value 9931.585655
## iter  60 value 9504.967033
## iter  70 value 7456.413606
## iter  80 value 7346.350085
## iter  90 value 7185.825713
## iter 100 value 6837.609049
## final  value 6837.609049 
## stopped after 100 iterations
## # weights:  127
## initial  value 10947.653512 
## iter  10 value 10565.599759
## iter  20 value 10565.346444
## final  value 10565.343643 
## converged
## # weights:  175
## initial  value 12562.792275 
## iter  10 value 10581.891694
## iter  20 value 10565.597974
## iter  30 value 10565.398640
## iter  40 value 9837.330022
## iter  50 value 8604.602772
## iter  60 value 8324.085764
## iter  70 value 8265.975235
## iter  80 value 8215.154818
## iter  90 value 8031.767727
## iter 100 value 6559.429334
## final  value 6559.429334 
## stopped after 100 iterations
## # weights:  55
## initial  value 12091.138715 
## iter  10 value 10565.489621
## final  value 10565.481389 
## converged
## # weights:  79
## initial  value 10948.498772 
## iter  10 value 10565.553257
## iter  20 value 10561.722071
## iter  30 value 9446.289462
## iter  40 value 8653.421577
## iter  50 value 7520.786166
## iter  60 value 6191.426205
## iter  70 value 5461.086846
## iter  80 value 5105.772953
## iter  90 value 4699.609358
## iter 100 value 4104.991564
## final  value 4104.991564 
## stopped after 100 iterations
## # weights:  127
## initial  value 11297.493816 
## iter  10 value 10565.690499
## final  value 10565.481422 
## converged
## # weights:  175
## initial  value 15744.526706 
## iter  10 value 10887.415329
## iter  20 value 10622.443925
## iter  30 value 10508.684409
## iter  40 value 10356.234609
## iter  50 value 10156.354859
## iter  60 value 10022.122421
## iter  70 value 9875.421121
## iter  80 value 9763.204353
## iter  90 value 9699.411015
## iter 100 value 9599.646582
## final  value 9599.646582 
## stopped after 100 iterations
## # weights:  55
## initial  value 11085.883751 
## iter  10 value 10568.434391
## final  value 10568.425176 
## converged
## # weights:  79
## initial  value 11402.514579 
## iter  10 value 10568.675371
## iter  20 value 10568.384696
## final  value 10568.361533 
## converged
## # weights:  127
## initial  value 12776.603845 
## iter  10 value 10568.388557
## final  value 10568.361711 
## converged
## # weights:  175
## initial  value 13575.991474 
## iter  10 value 10629.264602
## iter  20 value 10568.396183
## iter  30 value 10568.345257
## iter  30 value 10568.345226
## final  value 10568.344775 
## converged
## # weights:  55
## initial  value 11300.652842 
## iter  10 value 10569.444907
## iter  20 value 10568.643844
## iter  30 value 10568.449422
## final  value 10568.446405 
## converged
## # weights:  79
## initial  value 12327.081313 
## iter  10 value 10570.811152
## iter  20 value 10568.421578
## iter  30 value 10568.393462
## iter  30 value 10568.393459
## iter  30 value 10568.393459
## final  value 10568.393459 
## converged
## # weights:  127
## initial  value 14211.713692 
## iter  10 value 10581.226212
## iter  20 value 10568.597422
## iter  30 value 10568.447531
## iter  30 value 10568.447459
## final  value 10568.446363 
## converged
## # weights:  175
## initial  value 14851.340659 
## iter  10 value 10577.132131
## iter  20 value 9596.464505
## iter  30 value 8282.657737
## iter  40 value 8020.089683
## iter  50 value 7846.403268
## iter  60 value 7801.754065
## iter  70 value 7362.769690
## iter  80 value 6853.428218
## iter  90 value 6554.363085
## iter 100 value 6159.018784
## final  value 6159.018784 
## stopped after 100 iterations
## # weights:  55
## initial  value 12124.812782 
## iter  10 value 10569.746909
## final  value 10568.679044 
## converged
## # weights:  79
## initial  value 11363.058633 
## iter  10 value 10569.173877
## iter  20 value 10038.497779
## iter  30 value 9081.024130
## iter  40 value 8400.182547
## iter  50 value 8350.410005
## iter  60 value 7783.839803
## iter  70 value 7407.491913
## iter  80 value 6887.480314
## iter  90 value 6530.553255
## iter 100 value 6381.048657
## final  value 6381.048657 
## stopped after 100 iterations
## # weights:  127
## initial  value 12936.563546 
## iter  10 value 10568.462684
## final  value 10568.456574 
## converged
## # weights:  175
## initial  value 11122.074978 
## iter  10 value 10568.610674
## iter  20 value 10568.458683
## iter  30 value 10267.717962
## iter  40 value 9849.017755
## iter  50 value 9422.939730
## iter  60 value 8602.200958
## iter  70 value 8239.148831
## iter  80 value 8226.140786
## iter  90 value 8185.358026
## iter 100 value 6435.668670
## final  value 6435.668670 
## stopped after 100 iterations
## # weights:  175
## initial  value 21598.355112 
## iter  10 value 16175.758406
## iter  20 value 15852.429090
## iter  30 value 15851.588795
## iter  40 value 15851.549105
## iter  40 value 15851.548954
## iter  50 value 15850.833864
## iter  60 value 15850.698456
## final  value 15850.697746 
## converged
DryBean_TDA_KDE_5.50.5_n1_NN1Fit0
## Neural Network 
## 
## 8473 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5649, 5648, 5649 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa     
##   2     0.3    0.2203470  0.00000000
##   2     0.5    0.2407600  0.02995836
##   2     0.7    0.2685056  0.06716065
##   3     0.3    0.2767682  0.07490392
##   3     0.5    0.3277216  0.14633841
##   3     0.7    0.5010936  0.37107435
##   5     0.3    0.3685476  0.19664149
##   5     0.5    0.2203470  0.00000000
##   5     0.7    0.2203470  0.00000000
##   7     0.3    0.3385889  0.16995984
##   7     0.5    0.5721713  0.46807421
##   7     0.7    0.5213290  0.41059802
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 7 and decay = 0.5.
DryBean_TDA_KDE_5.50.5_n1_NN1Fit0$resample
##    Accuracy     Kappa Resample
## 1 0.5646018 0.4551806    Fold2
## 2 0.5984419 0.5078989    Fold1
## 3 0.5534703 0.4411431    Fold3
nb_tda_kde_5.50.5_n1_nn1_fit_re<-DryBean_TDA_KDE_5.50.5_n1_NN1Fit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n1_NN1Fit0)
## a 16-7-7 network with 175 weights
## options were - softmax modelling  decay=0.5
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h4  i1->h4  i2->h4  i3->h4  i4->h4  i5->h4  i6->h4  i7->h4  i8->h4  i9->h4 
##    0.00   -0.01    0.00    0.00    0.00    0.00    0.00   -0.01    0.00    0.00 
## i10->h4 i11->h4 i12->h4 i13->h4 i14->h4 i15->h4 i16->h4 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h5  i1->h5  i2->h5  i3->h5  i4->h5  i5->h5  i6->h5  i7->h5  i8->h5  i9->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h5 i11->h5 i12->h5 i13->h5 i14->h5 i15->h5 i16->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h6  i1->h6  i2->h6  i3->h6  i4->h6  i5->h6  i6->h6  i7->h6  i8->h6  i9->h6 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h6 i11->h6 i12->h6 i13->h6 i14->h6 i15->h6 i16->h6 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h7  i1->h7  i2->h7  i3->h7  i4->h7  i5->h7  i6->h7  i7->h7  i8->h7  i9->h7 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h7 i11->h7 i12->h7 i13->h7 i14->h7 i15->h7 i16->h7 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##  b->o1 h1->o1 h2->o1 h3->o1 h4->o1 h5->o1 h6->o1 h7->o1 
##   0.04   0.00   0.04   0.00   0.00   0.00   0.04   0.04 
##  b->o2 h1->o2 h2->o2 h3->o2 h4->o2 h5->o2 h6->o2 h7->o2 
##  -0.19   0.00  -0.19   0.00   0.00   0.00  -0.19  -0.19 
##  b->o3 h1->o3 h2->o3 h3->o3 h4->o3 h5->o3 h6->o3 h7->o3 
##   0.09   0.00   0.10   0.00   0.00   0.00   0.10   0.10 
##  b->o4 h1->o4 h2->o4 h3->o4 h4->o4 h5->o4 h6->o4 h7->o4 
##   0.07   0.00   0.07   0.00   0.00   0.00   0.07   0.07 
##  b->o5 h1->o5 h2->o5 h3->o5 h4->o5 h5->o5 h6->o5 h7->o5 
##   0.13   0.00   0.13   0.00   0.00   0.00   0.13   0.13 
##  b->o6 h1->o6 h2->o6 h3->o6 h4->o6 h5->o6 h6->o6 h7->o6 
##  -0.07   0.00  -0.07   0.00   0.00   0.00  -0.07  -0.07 
##  b->o7 h1->o7 h2->o7 h3->o7 h4->o7 h5->o7 h6->o7 h7->o7 
##  -0.08   0.00  -0.08   0.00   0.00   0.00  -0.08  -0.08
#vip(DryBean_TDA_KDE_5.50.5_n1_NN1Fit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n1_NN1Fit TDA-Assited NN")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n1_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n1_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n1_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
nb_tda_kde_5.50.5_n1_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ         396    156  489     1063   578   608  790
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.1417          
##                  95% CI : (0.1311, 0.1527)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.0000
## Specificity                  1.00000       1.00000      1.0000          1.0000
## Pos Pred Value                   NaN           NaN         NaN             NaN
## Neg Pred Value               0.90294       0.96176      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.0000
## Detection Prevalence         0.00000       0.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                1.0000        0.000      0.0000
## Specificity                0.0000        1.000      1.0000
## Pos Pred Value             0.1417          NaN         NaN
## Neg Pred Value                NaN        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1417        0.000      0.0000
## Detection Prevalence       1.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
nb_tda_kde_5.50.5_n1_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ         396    156  489     1063   578   608  790
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.1417          
##                  95% CI : (0.1311, 0.1527)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.0000
## Specificity                  1.00000       1.00000      1.0000          1.0000
## Pos Pred Value                   NaN           NaN         NaN             NaN
## Neg Pred Value               0.90294       0.96176      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.0000
## Detection Prevalence         0.00000       0.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                1.0000        0.000      0.0000
## Specificity                0.0000        1.000      1.0000
## Pos Pred Value             0.1417          NaN         NaN
## Neg Pred Value                NaN        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1417        0.000      0.0000
## Detection Prevalence       1.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
nb_tda_kde_5.50.5_n1_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.1416667      0.0000000      0.1311036      0.1527456      0.2605392 
## AccuracyPValue  McnemarPValue 
##      1.0000000            NaN
nb_tda_kde_5.50.5_n1_db_nn1_cf0_ov_acc<-nb_tda_kde_5.50.5_n1_db_nn1_cf0$overall[1]
nb_tda_kde_5.50.5_n1_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA           0           1            NaN      0.9029412        NA
## Class: BOMBAY             0           1            NaN      0.9617647        NA
## Class: CALI               0           1            NaN      0.8801471        NA
## Class: DERMASON           0           1            NaN      0.7394608        NA
## Class: HOROZ              1           0      0.1416667            NaN 0.1416667
## Class: SEKER              0           1            NaN      0.8509804        NA
## Class: SIRA               0           1            NaN      0.8063725        NA
##                 Recall        F1 Prevalence Detection Rate Detection Prevalence
## Class: BARBUNYA      0        NA 0.09705882      0.0000000                    0
## Class: BOMBAY        0        NA 0.03823529      0.0000000                    0
## Class: CALI          0        NA 0.11985294      0.0000000                    0
## Class: DERMASON      0        NA 0.26053922      0.0000000                    0
## Class: HOROZ         1 0.2481752 0.14166667      0.1416667                    1
## Class: SEKER         0        NA 0.14901961      0.0000000                    0
## Class: SIRA          0        NA 0.19362745      0.0000000                    0
##                 Balanced Accuracy
## Class: BARBUNYA               0.5
## Class: BOMBAY                 0.5
## Class: CALI                   0.5
## Class: DERMASON               0.5
## Class: HOROZ                  0.5
## Class: SEKER                  0.5
## Class: SIRA                   0.5
nb_tda_kde_5.50.5_n1_db_nn1_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n1_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold<-(db_nn1_fit_re - nb_tda_kde_5.50.5_n1_nn1_fit_re)
diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold
##      Accuracy
## 1  0.14100276
## 2 -0.06523450
## 3  0.01386769
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n1_3_fold
## $probLeft
## [1] 0.25
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n1_3_fold_odds.left<-bst_tda_kde_5.50.5_nn1.n1_3_fold$probLeft/bst_tda_kde_5.50.5_nn1.n1_3_fold$probRight
bst_tda_kde_5.50.5_nn1.n1_3_fold_odds.left
## [1] 0.5
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n1_3_fold
## $winLeft
## [1] 0.2772
## 
## $winRope
## [1] 0.05986667
## 
## $winRight
## [1] 0.6629333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n1_3_fold
## $left
## [1] 0.3117017
## 
## $rope
## [1] 0.08899445
## 
## $right
## [1] 0.5993038
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold))
#bf_tda_kde_5.50.5_nn1.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n1_3_fold)
## t = 0.49739, df = 2, p-value = 0.6682
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.2285877  0.2883450
## sample estimates:
##  mean of x 
## 0.02987865
### Test set diff
diff_drybean_tda_kde_5.50.5_nn1.n1_test<-(db_nn1_cf_ov_acc - nb_tda_kde_5.50.5_n1_db_nn1_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_nn1.n1_test
##  Accuracy 
## 0.4681373
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n1_test),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n1_test_odds.left<-bst_tda_kde_5.50.5_nn1.n1_test$probLeft/bst_tda_kde_5.50.5_nn1.n1_test$probRight
bst_tda_kde_5.50.5_nn1.n1_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n1_test),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1603333
## 
## $winRight
## [1] 0.8396667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n1_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nn1.n1_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n1_test)) #bf_tda_pca_5.50.5_nn1.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n1_test))

##Node2

#Neural Network 1
DryBean_TDA_KDE_5.50.5_n2_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n2.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  52
## initial  value 8556.797540 
## iter  10 value 8279.624882
## iter  20 value 6598.824199
## iter  30 value 6305.531459
## iter  40 value 6242.857077
## iter  50 value 6119.721398
## iter  60 value 5996.561258
## iter  70 value 5931.139467
## iter  80 value 5712.401497
## iter  90 value 5511.497037
## iter 100 value 4867.343811
## final  value 4867.343811 
## stopped after 100 iterations
## # weights:  75
## initial  value 10017.436483 
## iter  10 value 8279.126532
## iter  20 value 8212.426674
## iter  30 value 8002.608796
## iter  40 value 6266.657248
## iter  50 value 5861.176678
## iter  60 value 5566.949541
## iter  70 value 5481.181744
## iter  80 value 5457.009635
## iter  90 value 5414.394210
## iter 100 value 5320.855786
## final  value 5320.855786 
## stopped after 100 iterations
## # weights:  121
## initial  value 9910.648529 
## iter  10 value 8279.071006
## final  value 8279.061546 
## converged
## # weights:  167
## initial  value 10451.094004 
## iter  10 value 8279.093590
## iter  20 value 8180.589562
## iter  30 value 7705.175800
## iter  40 value 7605.193300
## iter  50 value 6519.057468
## iter  60 value 6497.787981
## iter  70 value 5916.344956
## iter  80 value 5165.611323
## iter  90 value 4944.099149
## iter 100 value 4670.162999
## final  value 4670.162999 
## stopped after 100 iterations
## # weights:  52
## initial  value 12460.881773 
## iter  10 value 9501.487927
## iter  20 value 8312.505094
## iter  30 value 6390.088366
## iter  40 value 6254.451593
## iter  50 value 5848.999732
## iter  60 value 4894.979821
## iter  70 value 4558.021749
## iter  80 value 4498.368804
## iter  90 value 4436.739144
## iter 100 value 4340.937975
## final  value 4340.937975 
## stopped after 100 iterations
## # weights:  75
## initial  value 9624.012249 
## iter  10 value 8281.307179
## iter  20 value 8279.314584
## iter  30 value 8279.291008
## iter  30 value 8279.290940
## iter  30 value 8279.290872
## final  value 8279.290872 
## converged
## # weights:  121
## initial  value 9643.699985 
## iter  10 value 8282.247008
## iter  20 value 8279.324647
## iter  30 value 8279.291171
## final  value 8279.290851 
## converged
## # weights:  167
## initial  value 9496.738010 
## iter  10 value 8280.238729
## iter  20 value 8279.118937
## final  value 8279.068432 
## converged
## # weights:  52
## initial  value 9417.552926 
## iter  10 value 8319.948539
## iter  20 value 8280.536582
## iter  30 value 8207.375165
## iter  40 value 7670.404322
## iter  50 value 7546.202876
## iter  60 value 6854.682866
## iter  70 value 5603.876571
## iter  80 value 5387.299231
## iter  90 value 4962.953258
## iter 100 value 4690.757640
## final  value 4690.757640 
## stopped after 100 iterations
## # weights:  75
## initial  value 8897.185190 
## iter  10 value 8279.751511
## final  value 8279.748874 
## converged
## # weights:  121
## initial  value 11203.197903 
## iter  10 value 8300.898629
## iter  20 value 8279.209543
## iter  30 value 8270.526167
## iter  40 value 8149.600654
## iter  50 value 6664.470253
## iter  60 value 6323.344840
## iter  70 value 4694.337756
## iter  80 value 4216.502549
## iter  90 value 4156.612240
## iter 100 value 3933.298242
## final  value 3933.298242 
## stopped after 100 iterations
## # weights:  167
## initial  value 11710.534120 
## iter  10 value 8279.306394
## iter  20 value 7615.140736
## iter  30 value 7534.152586
## iter  40 value 7024.252946
## iter  50 value 5796.410046
## iter  60 value 5176.412565
## iter  70 value 4989.753249
## iter  80 value 4441.219589
## iter  90 value 3704.837932
## iter 100 value 3399.020079
## final  value 3399.020079 
## stopped after 100 iterations
## # weights:  52
## initial  value 10554.368374 
## iter  10 value 8358.403765
## iter  20 value 8278.496491
## iter  30 value 8173.861229
## iter  40 value 7237.392537
## iter  50 value 7002.893507
## iter  60 value 5978.507746
## iter  70 value 3344.440488
## iter  80 value 2782.944170
## iter  90 value 2417.881325
## iter 100 value 2276.767709
## final  value 2276.767709 
## stopped after 100 iterations
## # weights:  75
## initial  value 9891.882718 
## iter  10 value 8276.001422
## final  value 8275.999850 
## converged
## # weights:  121
## initial  value 10360.497289 
## iter  10 value 8275.939631
## final  value 8275.937293 
## converged
## # weights:  167
## initial  value 10173.016202 
## iter  10 value 8320.319780
## iter  20 value 8278.466633
## iter  30 value 8277.922237
## iter  40 value 6496.284046
## iter  50 value 6284.827479
## iter  60 value 6114.007203
## iter  70 value 5523.089400
## iter  80 value 5070.041380
## iter  90 value 4135.547558
## iter 100 value 3662.258021
## final  value 3662.258021 
## stopped after 100 iterations
## # weights:  52
## initial  value 9056.782007 
## iter  10 value 8278.354986
## iter  20 value 8276.192090
## iter  30 value 8276.166379
## iter  30 value 8276.166365
## iter  30 value 8276.166363
## final  value 8276.166363 
## converged
## # weights:  75
## initial  value 11347.541623 
## iter  10 value 8370.879806
## iter  20 value 8279.793427
## iter  30 value 8278.105301
## iter  40 value 8276.189735
## iter  50 value 8272.856101
## iter  60 value 7973.197066
## iter  70 value 6259.859845
## iter  80 value 5962.820574
## iter  90 value 5920.042962
## iter 100 value 5883.614356
## final  value 5883.614356 
## stopped after 100 iterations
## # weights:  121
## initial  value 11345.935240 
## iter  10 value 8294.272906
## iter  20 value 8276.283313
## iter  30 value 8276.063716
## iter  40 value 8275.993346
## final  value 8275.957950 
## converged
## # weights:  167
## initial  value 10481.572948 
## iter  10 value 8294.669530
## iter  20 value 8276.247954
## iter  30 value 8276.021999
## iter  40 value 8275.803364
## iter  50 value 7901.830896
## iter  60 value 6207.066121
## iter  70 value 6078.317528
## iter  80 value 6043.871686
## iter  90 value 5992.725517
## iter 100 value 5934.513441
## final  value 5934.513441 
## stopped after 100 iterations
## # weights:  52
## initial  value 8781.201522 
## iter  10 value 8276.657878
## final  value 8276.628163 
## converged
## # weights:  75
## initial  value 9304.170538 
## iter  10 value 8276.722856
## iter  20 value 8276.624965
## final  value 8276.624142 
## converged
## # weights:  121
## initial  value 9934.907792 
## iter  10 value 8276.211732
## iter  20 value 8276.187370
## final  value 8276.187223 
## converged
## # weights:  167
## initial  value 10873.715541 
## iter  10 value 8277.145816
## iter  20 value 7289.105388
## iter  30 value 6046.558372
## iter  40 value 5974.914053
## iter  50 value 5558.132997
## iter  60 value 5202.733557
## iter  70 value 4672.990352
## iter  80 value 4482.058892
## iter  90 value 4457.524904
## iter 100 value 4437.704155
## final  value 4437.704155 
## stopped after 100 iterations
## # weights:  52
## initial  value 9442.925613 
## iter  10 value 8276.500008
## iter  20 value 8276.291949
## iter  30 value 8067.508394
## iter  40 value 7087.199427
## iter  50 value 6423.534584
## iter  60 value 6049.238469
## iter  70 value 5928.664256
## iter  80 value 5873.971112
## iter  90 value 5730.404031
## iter 100 value 5519.770201
## final  value 5519.770201 
## stopped after 100 iterations
## # weights:  75
## initial  value 10302.819421 
## iter  10 value 8282.658707
## iter  20 value 8222.476982
## iter  30 value 6725.028033
## iter  40 value 5995.028875
## iter  50 value 5891.091492
## iter  60 value 5849.710524
## iter  70 value 5840.506783
## iter  80 value 5836.707018
## iter  90 value 5792.903999
## iter 100 value 5454.205802
## final  value 5454.205802 
## stopped after 100 iterations
## # weights:  121
## initial  value 10713.913050 
## iter  10 value 8275.963922
## iter  20 value 8275.937220
## iter  20 value 8275.937146
## iter  20 value 8275.937142
## final  value 8275.937142 
## converged
## # weights:  167
## initial  value 12111.374322 
## iter  10 value 8388.716582
## iter  20 value 8276.509362
## iter  30 value 6883.112287
## iter  40 value 6662.634893
## iter  50 value 6618.938407
## iter  60 value 6128.732653
## iter  70 value 4617.497993
## iter  80 value 4175.655601
## iter  90 value 4085.361442
## iter 100 value 3952.972145
## final  value 3952.972145 
## stopped after 100 iterations
## # weights:  52
## initial  value 8833.220207 
## iter  10 value 8304.931535
## iter  20 value 8276.832713
## iter  30 value 8276.380053
## iter  40 value 8276.287657
## final  value 8276.213883 
## converged
## # weights:  75
## initial  value 9529.981318 
## iter  10 value 8288.414527
## iter  20 value 8276.346441
## iter  30 value 8276.168452
## iter  40 value 8276.161502
## iter  50 value 8276.064058
## final  value 8276.062140 
## converged
## # weights:  121
## initial  value 10728.391818 
## iter  10 value 8285.004765
## iter  20 value 8276.275219
## iter  30 value 8276.166482
## iter  40 value 8252.707410
## iter  50 value 7920.646148
## iter  60 value 6872.086826
## iter  70 value 6815.211211
## iter  80 value 6711.126940
## iter  90 value 6588.789633
## iter 100 value 5529.651982
## final  value 5529.651982 
## stopped after 100 iterations
## # weights:  167
## initial  value 11249.374020 
## iter  10 value 8286.713581
## iter  20 value 8276.210709
## iter  30 value 7894.554621
## iter  40 value 5248.269991
## iter  50 value 4712.548515
## iter  60 value 4507.393629
## iter  70 value 4423.484066
## iter  80 value 4304.432429
## iter  90 value 4219.297665
## iter 100 value 4084.950417
## final  value 4084.950417 
## stopped after 100 iterations
## # weights:  52
## initial  value 8969.503431 
## iter  10 value 8281.273629
## iter  20 value 8277.475032
## iter  30 value 8276.345315
## final  value 8276.333173 
## converged
## # weights:  75
## initial  value 9973.482086 
## iter  10 value 8276.631032
## final  value 8276.624058 
## converged
## # weights:  121
## initial  value 11565.747785 
## iter  10 value 8283.866391
## iter  20 value 8276.113673
## iter  30 value 8276.051979
## iter  40 value 8275.877463
## iter  50 value 6383.416083
## iter  60 value 6218.862022
## iter  70 value 6115.945912
## iter  80 value 5612.295198
## iter  90 value 5009.156735
## iter 100 value 3730.734991
## final  value 3730.734991 
## stopped after 100 iterations
## # weights:  167
## initial  value 8645.997056 
## iter  10 value 8222.015671
## iter  20 value 7987.332860
## iter  30 value 7474.042521
## iter  40 value 7273.091666
## iter  50 value 7155.246662
## iter  60 value 6918.447118
## iter  70 value 6222.874484
## iter  80 value 5869.183448
## iter  90 value 5711.488647
## iter 100 value 3835.679055
## final  value 3835.679055 
## stopped after 100 iterations
## # weights:  167
## initial  value 16686.819595 
## iter  10 value 12447.576631
## iter  20 value 12416.015396
## iter  30 value 12415.716637
## iter  40 value 10445.026952
## iter  50 value 10064.406715
## iter  60 value 9928.591552
## iter  70 value 9771.328477
## iter  80 value 7931.544590
## iter  90 value 7275.661317
## iter 100 value 6970.445498
## final  value 6970.445498 
## stopped after 100 iterations
DryBean_TDA_KDE_5.50.5_n2_NN1Fit0
## Neural Network 
## 
## 7582 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5056, 5054, 5054 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   2     0.3    0.6706414  0.5569011
##   2     0.5    0.4217263  0.1907494
##   2     0.7    0.4202747  0.1891611
##   3     0.3    0.4543885  0.2352842
##   3     0.5    0.3742853  0.1210137
##   3     0.7    0.2896334  0.0000000
##   5     0.3    0.2896334  0.0000000
##   5     0.5    0.3463317  0.1064236
##   5     0.7    0.5798279  0.4266460
##   7     0.3    0.7098207  0.6172923
##   7     0.5    0.5172178  0.3381399
##   7     0.7    0.7461163  0.6661104
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 7 and decay = 0.7.
DryBean_TDA_KDE_5.50.5_n2_NN1Fit0$resample
##    Accuracy     Kappa Resample
## 1 0.6843354 0.5708785    Fold2
## 2 0.7731591 0.7090646    Fold1
## 3 0.7808544 0.7183882    Fold3
nb_tda_kde_5.50.5_n2_nn1_fit_re<-DryBean_TDA_KDE_5.50.5_n2_NN1Fit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n2_NN1Fit0)
## a 16-7-6 network with 167 weights
## options were - softmax modelling  decay=0.7
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00    0.05    0.00    0.00    0.00    0.00    0.00    0.05    0.00    0.00 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##   -0.03   -1.21   -8.88   -2.52   -3.62   -0.03   -0.01    1.42   -3.10   -0.02 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##   -0.03   -0.03   -0.03    0.00    0.00   -0.03   -0.03 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##    0.00    0.01    0.00    0.00    0.00    0.00    0.00    0.01    0.00    0.00 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h4  i1->h4  i2->h4  i3->h4  i4->h4  i5->h4  i6->h4  i7->h4  i8->h4  i9->h4 
##    0.00    0.06    0.00    0.00    0.00    0.00    0.00    0.06    0.00    0.00 
## i10->h4 i11->h4 i12->h4 i13->h4 i14->h4 i15->h4 i16->h4 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h5  i1->h5  i2->h5  i3->h5  i4->h5  i5->h5  i6->h5  i7->h5  i8->h5  i9->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h5 i11->h5 i12->h5 i13->h5 i14->h5 i15->h5 i16->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h6  i1->h6  i2->h6  i3->h6  i4->h6  i5->h6  i6->h6  i7->h6  i8->h6  i9->h6 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h6 i11->h6 i12->h6 i13->h6 i14->h6 i15->h6 i16->h6 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h7  i1->h7  i2->h7  i3->h7  i4->h7  i5->h7  i6->h7  i7->h7  i8->h7  i9->h7 
##    0.02    0.76    2.86   -2.41    4.87   -0.02    0.00   -0.81    2.08    0.00 
## i10->h7 i11->h7 i12->h7 i13->h7 i14->h7 i15->h7 i16->h7 
##    0.02    0.03    0.03    0.00    0.00    0.03    0.02 
##  b->o1 h1->o1 h2->o1 h3->o1 h4->o1 h5->o1 h6->o1 h7->o1 
##   0.25   0.36   0.79   0.25   0.14   0.00   0.00  -3.01 
##  b->o2 h1->o2 h2->o2 h3->o2 h4->o2 h5->o2 h6->o2 h7->o2 
##   0.71   0.18   0.85   0.71   0.11   0.00   0.00  -5.71 
##  b->o3 h1->o3 h2->o3 h3->o3 h4->o3 h5->o3 h6->o3 h7->o3 
##  -0.30   0.38  -4.42  -0.30  -0.35   0.00   0.00   4.74 
##  b->o4 h1->o4 h2->o4 h3->o4 h4->o4 h5->o4 h6->o4 h7->o4 
##   0.71   0.30   1.01   0.71  -0.25   0.00   0.00  -2.34 
##  b->o5 h1->o5 h2->o5 h3->o5 h4->o5 h5->o5 h6->o5 h7->o5 
##  -0.84  -0.41  -1.18  -0.84   0.74   0.00   0.00   4.44 
##  b->o6 h1->o6 h2->o6 h3->o6 h4->o6 h5->o6 h6->o6 h7->o6 
##  -0.53  -0.80   2.95  -0.53  -0.39   0.00   0.00   1.88
#vip(DryBean_TDA_KDE_5.50.5_n2_NN1Fit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n2_NN1Fit TDA-Assited NN")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n2_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n2_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n2_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n2_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI          381    156  484        1   420     4   44
##   DERMASON        0      0    0      911    24   216   52
##   HOROZ           0      0    0        0     1     0    0
##   SEKER           0      0    0        1     0     2    0
##   SIRA           15      0    5      150   133   386  694
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5127          
##                  95% CI : (0.4973, 0.5282)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.4011          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.9898          0.8570
## Specificity                  1.00000       1.00000      0.7199          0.9032
## Pos Pred Value                   NaN           NaN      0.3248          0.7573
## Neg Pred Value               0.90294       0.96176      0.9981          0.9472
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.1186          0.2233
## Detection Prevalence         0.00000       0.00000      0.3652          0.2949
## Balanced Accuracy            0.50000       0.50000      0.8548          0.8801
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity             0.0017301    0.0032895      0.8785
## Specificity             1.0000000    0.9997120      0.7906
## Pos Pred Value          1.0000000    0.6666667      0.5018
## Neg Pred Value          0.8585438    0.8513613      0.9644
## Prevalence              0.1416667    0.1490196      0.1936
## Detection Rate          0.0002451    0.0004902      0.1701
## Detection Prevalence    0.0002451    0.0007353      0.3390
## Balanced Accuracy       0.5008651    0.5015007      0.8345
nb_tda_kde_5.50.5_n2_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI          381    156  484        1   420     4   44
##   DERMASON        0      0    0      911    24   216   52
##   HOROZ           0      0    0        0     1     0    0
##   SEKER           0      0    0        1     0     2    0
##   SIRA           15      0    5      150   133   386  694
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5127          
##                  95% CI : (0.4973, 0.5282)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.4011          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.9898          0.8570
## Specificity                  1.00000       1.00000      0.7199          0.9032
## Pos Pred Value                   NaN           NaN      0.3248          0.7573
## Neg Pred Value               0.90294       0.96176      0.9981          0.9472
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.1186          0.2233
## Detection Prevalence         0.00000       0.00000      0.3652          0.2949
## Balanced Accuracy            0.50000       0.50000      0.8548          0.8801
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity             0.0017301    0.0032895      0.8785
## Specificity             1.0000000    0.9997120      0.7906
## Pos Pred Value          1.0000000    0.6666667      0.5018
## Neg Pred Value          0.8585438    0.8513613      0.9644
## Prevalence              0.1416667    0.1490196      0.1936
## Detection Rate          0.0002451    0.0004902      0.1701
## Detection Prevalence    0.0002451    0.0007353      0.3390
## Balanced Accuracy       0.5008651    0.5015007      0.8345
nb_tda_kde_5.50.5_n2_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   5.127451e-01   4.011354e-01   4.972808e-01   5.281911e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##  2.042121e-257            NaN
nb_tda_kde_5.50.5_n2_db_nn1_cf0_ov_acc<-nb_tda_kde_5.50.5_n2_db_nn1_cf0$overall[1]
nb_tda_kde_5.50.5_n2_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA 0.000000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY   0.000000000   1.0000000            NaN      0.9617647        NA
## Class: CALI     0.989775051   0.7198552      0.3248322      0.9980695 0.3248322
## Class: DERMASON 0.857008467   0.9032151      0.7572735      0.9471672 0.7572735
## Class: HOROZ    0.001730104   1.0000000      1.0000000      0.8585438 1.0000000
## Class: SEKER    0.003289474   0.9997120      0.6666667      0.8513613 0.6666667
## Class: SIRA     0.878481013   0.7905775      0.5018077      0.9644049 0.5018077
##                      Recall          F1 Prevalence Detection Rate
## Class: BARBUNYA 0.000000000          NA 0.09705882   0.0000000000
## Class: BOMBAY   0.000000000          NA 0.03823529   0.0000000000
## Class: CALI     0.989775051 0.489135927 0.11985294   0.1186274510
## Class: DERMASON 0.857008467 0.804060018 0.26053922   0.2232843137
## Class: HOROZ    0.001730104 0.003454231 0.14166667   0.0002450980
## Class: SEKER    0.003289474 0.006546645 0.14901961   0.0004901961
## Class: SIRA     0.878481013 0.638748274 0.19362745   0.1700980392
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA         0.0000000000         0.5000000
## Class: BOMBAY           0.0000000000         0.5000000
## Class: CALI             0.3651960784         0.8548151
## Class: DERMASON         0.2948529412         0.8801118
## Class: HOROZ            0.0002450980         0.5008651
## Class: SEKER            0.0007352941         0.5015007
## Class: SIRA             0.3389705882         0.8345293
nb_tda_kde_5.50.5_n2_db_nn1_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n2_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold<-(db_nn1_fit_re - nb_tda_kde_5.50.5_n2_nn1_fit_re)
diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold
##      Accuracy
## 1  0.02126909
## 2 -0.23995172
## 3 -0.21351648
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n2_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n2_3_fold
## $probLeft
## [1] 0.5
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.25
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n2_3_fold_odds.left<-bst_tda_kde_5.50.5_nn1.n2_3_fold$probLeft/bst_tda_kde_5.50.5_nn1.n2_3_fold$probRight
bst_tda_kde_5.50.5_nn1.n2_3_fold_odds.left
## [1] 2
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n2_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n2_3_fold
## $winLeft
## [1] 0.8794
## 
## $winRope
## [1] 0.01496667
## 
## $winRight
## [1] 0.1056333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n2_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n2_3_fold
## $left
## [1] 0.8515766
## 
## $rope
## [1] 0.02379056
## 
## $right
## [1] 0.1246328
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold))
#bf_tda_kde_5.50.5_nn1.n2_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n2_3_fold)
## t = -1.7353, df = 2, p-value = 0.2248
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.5012692  0.2131364
## sample estimates:
##  mean of x 
## -0.1440664
### Test set diff
diff_drybean_tda_kde_5.50.5_nn1.n2_test<-(db_nn1_cf_ov_acc - nb_tda_kde_5.50.5_n2_db_nn1_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_nn1.n2_test
##   Accuracy 
## 0.09705882
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n2_test),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n2_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n2_test_odds.left<-bst_tda_kde_5.50.5_nn1.n2_test$probLeft/bst_tda_kde_5.50.5_nn1.n2_test$probRight
bst_tda_kde_5.50.5_nn1.n2_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n2_test),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1643
## 
## $winRight
## [1] 0.8357
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n2_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n2_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n2_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nn1.n2_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n2_test)) #bf_tda_pca_5.50.5_nn1.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n2_test))

##Node3

#Neural Network 1
DryBean_TDA_KDE_5.50.5_n3_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n3.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  52
## initial  value 4969.978330 
## iter  10 value 3209.964105
## iter  20 value 3177.749923
## iter  30 value 3120.615957
## iter  40 value 3103.737714
## iter  50 value 2602.039848
## iter  60 value 2282.226082
## iter  70 value 2071.494370
## iter  80 value 1972.288627
## iter  90 value 1658.636916
## iter 100 value 1338.700583
## final  value 1338.700583 
## stopped after 100 iterations
## # weights:  75
## initial  value 4768.323946 
## iter  10 value 3218.230308
## iter  20 value 3208.790199
## iter  30 value 3197.101123
## iter  40 value 2534.125828
## iter  50 value 2349.468334
## iter  60 value 2133.959554
## iter  70 value 2077.852058
## iter  80 value 2069.169026
## iter  90 value 1985.358537
## iter 100 value 1736.405320
## final  value 1736.405320 
## stopped after 100 iterations
## # weights:  121
## initial  value 6994.457579 
## iter  10 value 3226.320985
## iter  20 value 3218.684202
## iter  30 value 3217.030364
## iter  40 value 3216.370855
## iter  50 value 2477.359225
## iter  60 value 2341.398950
## iter  70 value 2278.545360
## iter  80 value 2150.475188
## iter  90 value 2094.190053
## iter 100 value 2032.248977
## final  value 2032.248977 
## stopped after 100 iterations
## # weights:  167
## initial  value 6750.289785 
## iter  10 value 3299.803199
## iter  20 value 3219.802074
## iter  30 value 3205.865098
## iter  40 value 3205.776043
## iter  50 value 3203.463996
## iter  60 value 2541.376338
## iter  70 value 2290.492178
## iter  80 value 2280.137994
## iter  90 value 1950.652845
## iter 100 value 1791.627453
## final  value 1791.627453 
## stopped after 100 iterations
## # weights:  52
## initial  value 5526.861372 
## iter  10 value 3324.074827
## iter  20 value 3258.434455
## iter  30 value 3224.745788
## iter  40 value 3210.762897
## iter  50 value 3208.858908
## iter  60 value 3208.823561
## final  value 3208.822684 
## converged
## # weights:  75
## initial  value 5347.478881 
## iter  10 value 3229.916593
## iter  20 value 3218.524211
## iter  30 value 3207.411904
## iter  40 value 2821.331417
## iter  50 value 2350.708567
## iter  60 value 1864.339228
## iter  70 value 1555.099637
## iter  80 value 1203.672709
## iter  90 value 1087.293173
## iter 100 value 1018.467097
## final  value 1018.467097 
## stopped after 100 iterations
## # weights:  121
## initial  value 5947.264068 
## iter  10 value 3303.343551
## iter  20 value 3263.877861
## iter  30 value 3222.245384
## iter  40 value 3206.747832
## iter  50 value 3206.634228
## iter  60 value 3206.598171
## iter  70 value 3206.594573
## iter  70 value 3206.594547
## iter  70 value 3206.594530
## final  value 3206.594530 
## converged
## # weights:  167
## initial  value 5515.691524 
## iter  10 value 3212.477546
## iter  20 value 3209.883947
## iter  30 value 3207.700952
## iter  40 value 3047.366315
## iter  50 value 2029.529104
## iter  60 value 2018.537335
## iter  70 value 1994.122916
## iter  80 value 1922.771602
## iter  90 value 1647.636170
## iter 100 value 1353.537010
## final  value 1353.537010 
## stopped after 100 iterations
## # weights:  52
## initial  value 5007.327551 
## iter  10 value 3222.650255
## iter  20 value 3211.514901
## iter  30 value 2532.008407
## iter  40 value 2216.554554
## iter  50 value 2084.466101
## iter  60 value 2040.677180
## iter  70 value 1935.597925
## iter  80 value 1761.614038
## iter  90 value 1759.642357
## final  value 1759.640793 
## converged
## # weights:  75
## initial  value 4975.946290 
## iter  10 value 3357.656151
## iter  20 value 3228.829232
## iter  30 value 3213.935789
## iter  40 value 3211.597534
## iter  50 value 3210.869936
## iter  60 value 3210.262732
## iter  70 value 3209.150099
## final  value 3209.098427 
## converged
## # weights:  121
## initial  value 5814.352037 
## iter  10 value 3257.865768
## iter  20 value 3253.451983
## iter  30 value 3209.784239
## iter  40 value 3169.969167
## iter  50 value 2573.581385
## iter  60 value 2210.138645
## iter  70 value 2199.780383
## iter  80 value 2199.319486
## iter  90 value 2196.916544
## iter 100 value 2181.472778
## final  value 2181.472778 
## stopped after 100 iterations
## # weights:  167
## initial  value 6512.357722 
## iter  10 value 3275.396856
## iter  20 value 3242.563805
## iter  30 value 3230.791725
## iter  40 value 3074.020325
## iter  50 value 2434.700204
## iter  60 value 2261.382858
## iter  70 value 1913.411177
## iter  80 value 1737.357132
## iter  90 value 1641.177894
## iter 100 value 1438.838709
## final  value 1438.838709 
## stopped after 100 iterations
## # weights:  52
## initial  value 7132.395340 
## iter  10 value 3208.136385
## iter  20 value 3206.804131
## iter  30 value 3206.100160
## iter  40 value 3205.358468
## iter  50 value 3205.137744
## iter  60 value 2575.993049
## iter  70 value 2412.751543
## iter  80 value 2170.632916
## iter  90 value 2149.865792
## iter 100 value 2148.163641
## final  value 2148.163641 
## stopped after 100 iterations
## # weights:  75
## initial  value 4978.498267 
## iter  10 value 3207.652263
## iter  20 value 3206.321629
## iter  30 value 3205.006879
## iter  40 value 3200.808643
## iter  50 value 3129.480188
## iter  60 value 2727.268784
## iter  70 value 2151.470262
## iter  80 value 1673.133154
## iter  90 value 1158.132555
## iter 100 value 1098.012345
## final  value 1098.012345 
## stopped after 100 iterations
## # weights:  121
## initial  value 5755.488742 
## iter  10 value 3205.486778
## iter  20 value 3204.010982
## iter  30 value 3194.302822
## iter  40 value 2701.322264
## iter  50 value 2260.851053
## iter  60 value 2058.520417
## iter  70 value 2045.610560
## iter  80 value 1963.880942
## iter  90 value 1915.948403
## iter 100 value 1851.504210
## final  value 1851.504210 
## stopped after 100 iterations
## # weights:  167
## initial  value 7009.434415 
## iter  10 value 3248.046296
## iter  20 value 3209.011048
## iter  30 value 3203.300303
## iter  40 value 3203.153061
## iter  50 value 3202.978325
## final  value 3202.949223 
## converged
## # weights:  52
## initial  value 5376.828592 
## iter  10 value 3225.866314
## iter  20 value 3213.137427
## iter  30 value 3198.092421
## iter  40 value 3003.335997
## iter  50 value 2531.059993
## iter  60 value 2469.866515
## iter  70 value 2412.505864
## iter  80 value 2151.210013
## iter  90 value 1625.980984
## iter 100 value 1459.244503
## final  value 1459.244503 
## stopped after 100 iterations
## # weights:  75
## initial  value 5787.321303 
## iter  10 value 3211.168853
## iter  20 value 3208.122170
## iter  30 value 3207.738716
## iter  40 value 3206.078023
## final  value 3206.065961 
## converged
## # weights:  121
## initial  value 4565.007791 
## iter  10 value 3223.055506
## iter  20 value 3208.778015
## iter  30 value 3207.429969
## iter  40 value 3204.853135
## iter  50 value 3204.553150
## iter  60 value 3204.525540
## iter  70 value 3204.511848
## final  value 3204.511135 
## converged
## # weights:  167
## initial  value 4975.723823 
## iter  10 value 3264.337580
## iter  20 value 3220.391501
## iter  30 value 3204.662142
## iter  40 value 3204.502547
## iter  50 value 3204.288237
## iter  60 value 3203.959286
## iter  70 value 3203.627499
## iter  80 value 3165.937025
## iter  90 value 2312.757614
## iter 100 value 2261.152312
## final  value 2261.152312 
## stopped after 100 iterations
## # weights:  52
## initial  value 5817.418350 
## iter  10 value 3218.771371
## iter  20 value 3153.492815
## iter  30 value 2666.724047
## iter  40 value 2559.296979
## iter  50 value 2465.385156
## iter  60 value 2356.407787
## iter  70 value 2314.120129
## iter  80 value 2135.265566
## iter  90 value 1865.980224
## iter 100 value 1582.018866
## final  value 1582.018866 
## stopped after 100 iterations
## # weights:  75
## initial  value 6925.396894 
## iter  10 value 3286.827215
## iter  20 value 3259.239790
## iter  30 value 2536.659653
## iter  40 value 2164.543617
## iter  50 value 2066.840440
## iter  60 value 1996.407614
## iter  70 value 1946.039219
## iter  80 value 1909.763783
## iter  90 value 1779.259488
## iter 100 value 1607.939678
## final  value 1607.939678 
## stopped after 100 iterations
## # weights:  121
## initial  value 6165.810721 
## iter  10 value 3566.002920
## iter  20 value 3246.726347
## iter  30 value 3220.428777
## iter  40 value 3218.187981
## iter  50 value 3217.520696
## iter  60 value 3011.856135
## iter  70 value 2241.695507
## iter  80 value 2195.549969
## iter  90 value 1910.288934
## iter 100 value 1642.895088
## final  value 1642.895088 
## stopped after 100 iterations
## # weights:  167
## initial  value 5094.142376 
## iter  10 value 3210.514595
## iter  20 value 3210.058424
## iter  30 value 2639.038421
## iter  40 value 2292.989528
## iter  50 value 2221.219373
## iter  60 value 2100.222237
## iter  70 value 1370.245375
## iter  80 value 1280.786181
## iter  90 value 1240.034398
## iter 100 value 1201.875976
## final  value 1201.875976 
## stopped after 100 iterations
## # weights:  52
## initial  value 4558.939341 
## iter  10 value 3204.786183
## iter  20 value 3199.073017
## iter  30 value 2937.389981
## iter  40 value 2905.709589
## iter  50 value 2609.375588
## iter  60 value 2422.924594
## iter  70 value 2154.208386
## iter  80 value 2098.807759
## iter  90 value 2074.336754
## iter 100 value 2029.296401
## final  value 2029.296401 
## stopped after 100 iterations
## # weights:  75
## initial  value 4663.912738 
## iter  10 value 3202.357680
## iter  20 value 3200.592623
## iter  30 value 3199.516677
## iter  40 value 2901.861809
## iter  50 value 2629.615695
## iter  60 value 2426.119609
## iter  70 value 2282.281043
## iter  80 value 2101.800032
## iter  90 value 2080.215681
## iter 100 value 2055.259059
## final  value 2055.259059 
## stopped after 100 iterations
## # weights:  121
## initial  value 6694.658852 
## iter  10 value 3568.353442
## iter  20 value 3215.635290
## iter  30 value 3002.269470
## iter  40 value 2562.862081
## iter  50 value 2483.041267
## iter  60 value 2270.828245
## iter  70 value 2156.379372
## iter  80 value 1551.963280
## iter  90 value 1436.203595
## iter 100 value 1379.040178
## final  value 1379.040178 
## stopped after 100 iterations
## # weights:  167
## initial  value 5884.920265 
## iter  10 value 3263.011747
## iter  20 value 3242.468560
## iter  30 value 3026.184884
## iter  40 value 2400.748482
## iter  50 value 1869.598335
## iter  60 value 1297.107833
## iter  70 value 1224.136063
## iter  80 value 1188.718108
## iter  90 value 1149.356518
## iter 100 value 1138.044625
## final  value 1138.044625 
## stopped after 100 iterations
## # weights:  52
## initial  value 5746.375140 
## iter  10 value 3204.586556
## iter  20 value 3203.367424
## iter  30 value 2835.532229
## iter  40 value 2132.790297
## iter  50 value 2090.760053
## iter  60 value 2005.730023
## iter  70 value 1910.977213
## iter  80 value 1599.748714
## iter  90 value 1256.125717
## iter 100 value 1064.163094
## final  value 1064.163094 
## stopped after 100 iterations
## # weights:  75
## initial  value 4986.270698 
## iter  10 value 3222.914866
## iter  20 value 3212.079649
## iter  30 value 3180.213478
## iter  40 value 2973.227614
## iter  50 value 2783.463514
## iter  60 value 2369.660407
## iter  70 value 2339.900527
## iter  80 value 2309.213379
## iter  90 value 1548.301725
## iter 100 value 1323.077052
## final  value 1323.077052 
## stopped after 100 iterations
## # weights:  121
## initial  value 6547.087385 
## iter  10 value 3214.011311
## iter  20 value 3210.142264
## iter  30 value 2498.570812
## iter  40 value 2297.796104
## iter  50 value 2263.093903
## iter  60 value 2097.329159
## iter  70 value 2046.668950
## iter  80 value 1992.017942
## iter  90 value 1930.505396
## iter 100 value 1792.799062
## final  value 1792.799062 
## stopped after 100 iterations
## # weights:  167
## initial  value 4684.515706 
## iter  10 value 3269.861268
## iter  20 value 3206.278317
## iter  30 value 3199.944293
## iter  40 value 3199.175602
## iter  50 value 3199.072853
## iter  60 value 3198.841774
## iter  70 value 3198.587109
## iter  80 value 3198.319484
## iter  90 value 3178.000195
## iter 100 value 2791.155216
## final  value 2791.155216 
## stopped after 100 iterations
## # weights:  52
## initial  value 6515.484485 
## iter  10 value 3239.279085
## iter  20 value 2794.431933
## iter  30 value 2668.544819
## iter  40 value 2490.410065
## iter  50 value 1976.884036
## iter  60 value 1488.038850
## iter  70 value 1269.840333
## iter  80 value 1163.603500
## iter  90 value 1103.683873
## iter 100 value 1052.674018
## final  value 1052.674018 
## stopped after 100 iterations
## # weights:  75
## initial  value 5286.435465 
## iter  10 value 3240.794527
## iter  20 value 3206.244899
## iter  30 value 3203.393345
## iter  40 value 3020.631234
## iter  50 value 2793.961578
## iter  60 value 2537.222158
## iter  70 value 2222.256966
## iter  80 value 1928.377165
## iter  90 value 1564.694548
## iter 100 value 1354.159807
## final  value 1354.159807 
## stopped after 100 iterations
## # weights:  121
## initial  value 4907.230200 
## iter  10 value 3209.074842
## iter  20 value 3206.353658
## iter  30 value 3202.119353
## iter  40 value 3201.595631
## iter  50 value 3173.860094
## iter  60 value 2935.861469
## iter  70 value 2607.174746
## iter  80 value 2107.255502
## iter  90 value 1877.865394
## iter 100 value 1444.543010
## final  value 1444.543010 
## stopped after 100 iterations
## # weights:  167
## initial  value 4681.160404 
## iter  10 value 3238.217264
## iter  20 value 3214.840187
## iter  30 value 3203.448259
## iter  40 value 3129.524036
## iter  50 value 2710.621977
## iter  60 value 2244.415136
## iter  70 value 2213.319733
## iter  80 value 2207.217669
## iter  90 value 2102.199359
## iter 100 value 1252.378460
## final  value 1252.378460 
## stopped after 100 iterations
## # weights:  167
## initial  value 8574.336580 
## iter  10 value 5078.857550
## iter  20 value 4812.122155
## iter  30 value 4811.215353
## iter  40 value 4745.905009
## iter  50 value 3725.694534
## iter  60 value 3454.730277
## iter  70 value 3309.245039
## iter  80 value 2975.425516
## iter  90 value 2433.686206
## iter 100 value 2068.920154
## final  value 2068.920154 
## stopped after 100 iterations
DryBean_TDA_KDE_5.50.5_n3_NN1Fit0
## Neural Network 
## 
## 4149 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 2766, 2767, 2765 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   2     0.3    0.7334305  0.5791695
##   2     0.5    0.7160733  0.5460594
##   2     0.7    0.8365712  0.7511677
##   3     0.3    0.7797566  0.6554878
##   3     0.5    0.7193306  0.5511962
##   3     0.7    0.6934120  0.5119131
##   5     0.3    0.7541269  0.6158524
##   5     0.5    0.5111167  0.2104419
##   5     0.7    0.7929564  0.6754986
##   7     0.3    0.7075162  0.5327891
##   7     0.5    0.7083867  0.5340708
##   7     0.7    0.8756300  0.8108256
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 7 and decay = 0.7.
DryBean_TDA_KDE_5.50.5_n3_NN1Fit0$resample
##    Accuracy     Kappa Resample
## 1 0.8791606 0.8159733    Fold2
## 2 0.8575560 0.7837989    Fold1
## 3 0.8901734 0.8327046    Fold3
nb_tda_kde_5.50.5_n3_nn1_fit_re<-DryBean_TDA_KDE_5.50.5_n3_NN1Fit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n3_NN1Fit0)
## a 16-7-6 network with 167 weights
## options were - softmax modelling  decay=0.7
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.02    0.62    4.81    9.54   -2.75    0.11    0.08   -0.77    3.00    0.08 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.02    0.04   -0.01    0.00    0.00   -0.03    0.03 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.00   -0.01    0.00    0.00    0.00    0.00    0.00   -0.01    0.00    0.00 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##    0.00    0.02    0.00    0.00    0.00    0.00    0.00    0.03    0.00    0.00 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h4  i1->h4  i2->h4  i3->h4  i4->h4  i5->h4  i6->h4  i7->h4  i8->h4  i9->h4 
##    0.00   -1.55   -0.10    4.25   -4.86    0.05    0.03    1.55   -1.09   -0.01 
## i10->h4 i11->h4 i12->h4 i13->h4 i14->h4 i15->h4 i16->h4 
##   -0.01   -0.01   -0.02    0.00    0.00   -0.03    0.00 
##   b->h5  i1->h5  i2->h5  i3->h5  i4->h5  i5->h5  i6->h5  i7->h5  i8->h5  i9->h5 
##    0.00   -0.05    0.00    0.00    0.00    0.00    0.00   -0.05    0.00    0.00 
## i10->h5 i11->h5 i12->h5 i13->h5 i14->h5 i15->h5 i16->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h6  i1->h6  i2->h6  i3->h6  i4->h6  i5->h6  i6->h6  i7->h6  i8->h6  i9->h6 
##   -0.01   -0.24   -0.15    0.58   -3.06   -0.01    0.10    0.25    0.40   -0.04 
## i10->h6 i11->h6 i12->h6 i13->h6 i14->h6 i15->h6 i16->h6 
##   -0.01   -0.03   -0.04    0.00    0.00   -0.07   -0.02 
##   b->h7  i1->h7  i2->h7  i3->h7  i4->h7  i5->h7  i6->h7  i7->h7  i8->h7  i9->h7 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h7 i11->h7 i12->h7 i13->h7 i14->h7 i15->h7 i16->h7 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##  b->o1 h1->o1 h2->o1 h3->o1 h4->o1 h5->o1 h6->o1 h7->o1 
##  -0.96  -0.82   1.41  -0.97  -0.34  -0.06  -0.06   0.01 
##  b->o2 h1->o2 h2->o2 h3->o2 h4->o2 h5->o2 h6->o2 h7->o2 
##  -0.72  -0.43  -1.74  -0.73  -0.30  -0.44  -0.16   0.02 
##  b->o3 h1->o3 h2->o3 h3->o3 h4->o3 h5->o3 h6->o3 h7->o3 
##   0.11   3.70   0.02   0.12   0.66   0.15  -0.32   0.00 
##  b->o4 h1->o4 h2->o4 h3->o4 h4->o4 h5->o4 h6->o4 h7->o4 
##  -0.98  -0.21  -0.26  -0.98   1.50   1.43   0.73   0.00 
##  b->o5 h1->o5 h2->o5 h3->o5 h4->o5 h5->o5 h6->o5 h7->o5 
##   2.01  -1.80   0.44   2.02  -1.49  -0.39  -2.14  -0.01 
##  b->o6 h1->o6 h2->o6 h3->o6 h4->o6 h5->o6 h6->o6 h7->o6 
##   0.54  -0.43   0.11   0.54  -0.02  -0.69   1.94  -0.01
#vip(DryBean_TDA_KDE_5.50.5_n3_NN1Fit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n3_NN1Fit TDA-Assited NN")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n3_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n3_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n3_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n3_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0      945   105    62   94
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0       37     0   485   13
##   SIRA          396    156  489       81   473    61  683
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5179          
##                  95% CI : (0.5024, 0.5333)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.3916          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.8890
## Specificity                  1.00000       1.00000      1.0000          0.9135
## Pos Pred Value                   NaN           NaN         NaN          0.7836
## Neg Pred Value               0.90294       0.96176      0.8801          0.9589
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2316
## Detection Prevalence         0.00000       0.00000      0.0000          0.2956
## Balanced Accuracy            0.50000       0.50000      0.5000          0.9012
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000       0.7977      0.8646
## Specificity                1.0000       0.9856      0.4967
## Pos Pred Value                NaN       0.9065      0.2920
## Neg Pred Value             0.8583       0.9653      0.9385
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.0000       0.1189      0.1674
## Detection Prevalence       0.0000       0.1311      0.5733
## Balanced Accuracy          0.5000       0.8916      0.6806
nb_tda_kde_5.50.5_n3_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0      945   105    62   94
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0       37     0   485   13
##   SIRA          396    156  489       81   473    61  683
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5179          
##                  95% CI : (0.5024, 0.5333)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.3916          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.8890
## Specificity                  1.00000       1.00000      1.0000          0.9135
## Pos Pred Value                   NaN           NaN         NaN          0.7836
## Neg Pred Value               0.90294       0.96176      0.8801          0.9589
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2316
## Detection Prevalence         0.00000       0.00000      0.0000          0.2956
## Balanced Accuracy            0.50000       0.50000      0.5000          0.9012
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000       0.7977      0.8646
## Specificity                1.0000       0.9856      0.4967
## Pos Pred Value                NaN       0.9065      0.2920
## Neg Pred Value             0.8583       0.9653      0.9385
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.0000       0.1189      0.1674
## Detection Prevalence       0.0000       0.1311      0.5733
## Balanced Accuracy          0.5000       0.8916      0.6806
nb_tda_kde_5.50.5_n3_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   5.178922e-01   3.916187e-01   5.024290e-01   5.333296e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##  1.709603e-267            NaN
nb_tda_kde_5.50.5_n3_db_nn1_cf0_ov_acc<-nb_tda_kde_5.50.5_n3_db_nn1_cf0$overall[1]
nb_tda_kde_5.50.5_n3_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.0000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.0000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON   0.8889934   0.9134902      0.7835821      0.9589422 0.7835821
## Class: HOROZ      0.0000000   1.0000000            NaN      0.8583333        NA
## Class: SEKER      0.7976974   0.9855991      0.9065421      0.9653032 0.9065421
## Class: SIRA       0.8645570   0.4966565      0.2920051      0.9385411 0.2920051
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.0000000        NA 0.09705882      0.0000000
## Class: BOMBAY   0.0000000        NA 0.03823529      0.0000000
## Class: CALI     0.0000000        NA 0.11985294      0.0000000
## Class: DERMASON 0.8889934 0.8329661 0.26053922      0.2316176
## Class: HOROZ    0.0000000        NA 0.14166667      0.0000000
## Class: SEKER    0.7976974 0.8486439 0.14901961      0.1188725
## Class: SIRA     0.8645570 0.4365612 0.19362745      0.1674020
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA            0.0000000         0.5000000
## Class: BOMBAY              0.0000000         0.5000000
## Class: CALI                0.0000000         0.5000000
## Class: DERMASON            0.2955882         0.9012418
## Class: HOROZ               0.0000000         0.5000000
## Class: SEKER               0.1311275         0.8916482
## Class: SIRA                0.5732843         0.6806067
nb_tda_kde_5.50.5_n3_db_nn1_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n3_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold<-(db_nn1_fit_re - nb_tda_kde_5.50.5_n3_nn1_fit_re)
diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold
##     Accuracy
## 1 -0.1735561
## 2 -0.3243486
## 3 -0.3228355
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n3_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n3_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n3_3_fold_odds.left<-bst_tda_kde_5.50.5_nn1.n3_3_fold$probLeft/bst_tda_kde_5.50.5_nn1.n3_3_fold$probRight
bst_tda_kde_5.50.5_nn1.n3_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n3_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n3_3_fold
## $winLeft
## [1] 0.9903333
## 
## $winRope
## [1] 0.009666667
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n3_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n3_3_fold
## $left
## [1] 0.9775979
## 
## $rope
## [1] 0.002872362
## 
## $right
## [1] 0.01952977
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold))
#bf_tda_kde_5.50.5_nn1.n3_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n3_3_fold)
## t = -5.4701, df = 2, p-value = 0.03183
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.48877244 -0.05838768
## sample estimates:
##  mean of x 
## -0.2735801
### Test set diff
diff_drybean_tda_kde_5.50.5_nn1.n3_test<-(db_nn1_cf_ov_acc - nb_tda_kde_5.50.5_n3_db_nn1_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_nn1.n3_test
##   Accuracy 
## 0.09191176
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n3_test),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n3_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n3_test_odds.left<-bst_tda_kde_5.50.5_nn1.n3_test$probLeft/bst_tda_kde_5.50.5_nn1.n3_test$probRight
bst_tda_kde_5.50.5_nn1.n3_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n3_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n3_test),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n3_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1586667
## 
## $winRight
## [1] 0.8413333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n3_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n3_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n3_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nn1.n3_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n3_test)) #bf_tda_pca_5.50.5_nn1.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n3_test))

##Node4

#Neural Network 1
DryBean_TDA_KDE_5.50.5_n4_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n4.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  46
## initial  value 1711.865013 
## iter  10 value 1410.151303
## iter  20 value 1312.981656
## iter  30 value 869.937500
## iter  40 value 706.326657
## iter  50 value 665.232862
## iter  60 value 633.023238
## iter  70 value 620.280055
## iter  80 value 609.643320
## iter  90 value 594.173366
## final  value 594.148556 
## converged
## # weights:  67
## initial  value 2177.975098 
## iter  10 value 1405.900831
## iter  20 value 1363.901707
## iter  30 value 1215.539384
## iter  40 value 1092.582311
## iter  50 value 847.309124
## iter  60 value 736.642930
## iter  70 value 714.830862
## iter  80 value 667.066295
## iter  90 value 604.653230
## iter 100 value 576.018864
## final  value 576.018864 
## stopped after 100 iterations
## # weights:  109
## initial  value 1628.555046 
## iter  10 value 1405.736635
## iter  20 value 1404.015223
## iter  30 value 1403.890476
## iter  40 value 1392.305397
## iter  50 value 1349.041713
## iter  60 value 1108.131072
## iter  70 value 924.034051
## iter  80 value 724.003613
## iter  90 value 646.316131
## iter 100 value 613.900550
## final  value 613.900550 
## stopped after 100 iterations
## # weights:  151
## initial  value 2461.616664 
## iter  10 value 1405.997035
## iter  20 value 1403.882363
## final  value 1403.793911 
## converged
## # weights:  46
## initial  value 2097.721965 
## iter  10 value 1415.041269
## iter  20 value 1406.934972
## iter  30 value 1406.648133
## iter  40 value 1406.641446
## iter  50 value 1375.329702
## iter  60 value 1332.386260
## iter  70 value 1267.143124
## iter  80 value 957.454435
## iter  90 value 834.157511
## iter 100 value 804.807143
## final  value 804.807143 
## stopped after 100 iterations
## # weights:  67
## initial  value 1777.670696 
## iter  10 value 1408.235141
## iter  20 value 1406.550345
## iter  30 value 1196.313444
## iter  40 value 912.550545
## iter  50 value 829.365510
## iter  60 value 754.469162
## iter  70 value 672.805206
## iter  80 value 618.225280
## iter  90 value 608.862322
## iter 100 value 593.904818
## final  value 593.904818 
## stopped after 100 iterations
## # weights:  109
## initial  value 1970.643224 
## iter  10 value 1416.274064
## iter  20 value 1405.570549
## iter  30 value 1404.960786
## iter  40 value 1404.554677
## iter  50 value 1404.350181
## iter  60 value 1404.345722
## iter  70 value 1404.335666
## final  value 1404.335223 
## converged
## # weights:  151
## initial  value 2458.500221 
## iter  10 value 1414.928662
## iter  20 value 1407.726928
## iter  30 value 1386.479127
## iter  40 value 1131.433144
## iter  50 value 842.381856
## iter  60 value 816.174918
## iter  70 value 790.417366
## iter  80 value 774.460270
## iter  90 value 763.561740
## iter 100 value 762.971086
## final  value 762.971086 
## stopped after 100 iterations
## # weights:  46
## initial  value 1563.689212 
## iter  10 value 1408.606751
## iter  20 value 1407.959539
## iter  30 value 1407.951090
## iter  40 value 1407.898128
## iter  50 value 1407.229962
## iter  60 value 1391.760584
## iter  70 value 1281.471173
## iter  80 value 1208.229396
## iter  90 value 1174.912863
## iter 100 value 1151.286079
## final  value 1151.286079 
## stopped after 100 iterations
## # weights:  67
## initial  value 1779.838415 
## iter  10 value 1406.429648
## iter  20 value 1406.420484
## iter  30 value 1406.016224
## iter  40 value 1405.634479
## final  value 1405.626836 
## converged
## # weights:  109
## initial  value 1540.035371 
## iter  10 value 1413.830765
## iter  20 value 1408.004695
## iter  30 value 1406.614031
## iter  40 value 1404.526111
## iter  50 value 1142.439615
## iter  60 value 922.823426
## iter  70 value 867.452444
## iter  80 value 848.823899
## iter  90 value 811.911889
## iter 100 value 799.144129
## final  value 799.144129 
## stopped after 100 iterations
## # weights:  151
## initial  value 1872.859619 
## iter  10 value 1405.740191
## iter  20 value 1405.157420
## iter  30 value 1404.791395
## iter  40 value 1404.676988
## iter  50 value 1250.429186
## iter  60 value 1128.748171
## iter  70 value 951.310440
## iter  80 value 817.717083
## iter  90 value 796.842058
## iter 100 value 779.449223
## final  value 779.449223 
## stopped after 100 iterations
## # weights:  46
## initial  value 2378.294149 
## iter  10 value 1405.397029
## iter  20 value 1405.273548
## iter  30 value 1249.869510
## iter  40 value 1115.712458
## iter  50 value 887.923343
## iter  60 value 833.673131
## iter  70 value 806.948572
## iter  80 value 796.964009
## iter  90 value 794.140611
## iter 100 value 766.246715
## final  value 766.246715 
## stopped after 100 iterations
## # weights:  67
## initial  value 2008.976245 
## iter  10 value 1405.360307
## iter  20 value 1404.591013
## iter  30 value 1404.576393
## final  value 1404.576293 
## converged
## # weights:  109
## initial  value 1753.692532 
## iter  10 value 1410.997445
## iter  20 value 1405.003287
## iter  30 value 1403.754626
## iter  40 value 1323.602037
## iter  50 value 1172.956433
## iter  60 value 940.144850
## iter  70 value 802.622142
## iter  80 value 774.409781
## iter  90 value 770.521520
## iter 100 value 758.275708
## final  value 758.275708 
## stopped after 100 iterations
## # weights:  151
## initial  value 1510.428290 
## iter  10 value 1403.864307
## iter  20 value 1403.850731
## final  value 1403.850661 
## converged
## # weights:  46
## initial  value 1945.851951 
## iter  10 value 1410.100335
## iter  20 value 1406.729296
## iter  30 value 1374.215401
## iter  40 value 1096.579929
## iter  50 value 868.034401
## iter  60 value 799.629437
## iter  70 value 755.269992
## iter  80 value 704.442267
## iter  90 value 685.109483
## iter 100 value 655.262540
## final  value 655.262540 
## stopped after 100 iterations
## # weights:  67
## initial  value 2173.906205 
## iter  10 value 1414.995832
## iter  20 value 1406.896726
## iter  30 value 1406.645732
## iter  40 value 1406.629848
## iter  50 value 1405.421837
## iter  60 value 1405.194250
## iter  70 value 1372.269529
## iter  80 value 860.619201
## iter  90 value 818.846319
## iter 100 value 784.657503
## final  value 784.657503 
## stopped after 100 iterations
## # weights:  109
## initial  value 1947.370857 
## iter  10 value 1413.202404
## iter  20 value 1407.169849
## iter  30 value 1405.209291
## iter  40 value 1405.082090
## iter  50 value 1404.680813
## iter  60 value 1362.328046
## iter  70 value 1324.166410
## iter  80 value 1061.408923
## iter  90 value 902.608538
## iter 100 value 824.665734
## final  value 824.665734 
## stopped after 100 iterations
## # weights:  151
## initial  value 2414.734218 
## iter  10 value 1314.489082
## iter  20 value 1116.208427
## iter  30 value 1050.060915
## iter  40 value 920.167352
## iter  50 value 846.685294
## iter  60 value 815.477982
## iter  70 value 728.887118
## iter  80 value 677.116142
## iter  90 value 645.090689
## iter 100 value 621.999209
## final  value 621.999209 
## stopped after 100 iterations
## # weights:  46
## initial  value 2506.161810 
## iter  10 value 1406.422707
## iter  20 value 1406.420649
## final  value 1406.420601 
## converged
## # weights:  67
## initial  value 1917.352674 
## iter  10 value 1411.877725
## iter  20 value 1409.787337
## iter  30 value 1337.813977
## iter  40 value 961.094897
## iter  50 value 837.558020
## iter  60 value 826.187459
## iter  70 value 801.145019
## iter  80 value 785.229234
## iter  90 value 771.778944
## final  value 771.745950 
## converged
## # weights:  109
## initial  value 1949.450429 
## iter  10 value 1415.715749
## iter  20 value 1409.597042
## iter  30 value 1406.261196
## iter  40 value 1295.651499
## iter  50 value 1125.940204
## iter  60 value 859.008726
## iter  70 value 832.522930
## iter  80 value 816.494127
## iter  90 value 746.843918
## iter 100 value 715.664424
## final  value 715.664424 
## stopped after 100 iterations
## # weights:  151
## initial  value 1523.585544 
## iter  10 value 1410.122590
## iter  20 value 1404.942577
## iter  30 value 1404.811209
## iter  40 value 1404.589926
## iter  50 value 1404.576550
## iter  50 value 1404.576540
## iter  60 value 1404.571593
## iter  70 value 1404.421722
## iter  80 value 1404.395464
## final  value 1404.395196 
## converged
## # weights:  46
## initial  value 2043.209824 
## iter  10 value 1409.979127
## iter  20 value 1409.943730
## iter  30 value 1408.683814
## iter  40 value 1259.522159
## iter  50 value 1252.295744
## iter  60 value 1079.747052
## iter  70 value 835.436834
## iter  80 value 801.319515
## iter  90 value 781.486227
## iter 100 value 755.631657
## final  value 755.631657 
## stopped after 100 iterations
## # weights:  67
## initial  value 1835.760337 
## iter  10 value 1409.550851
## iter  20 value 1409.184363
## final  value 1408.990786 
## converged
## # weights:  109
## initial  value 1954.385964 
## iter  10 value 1413.922047
## iter  20 value 1407.203945
## iter  30 value 1177.112264
## iter  40 value 992.144778
## iter  50 value 976.006050
## iter  60 value 867.733247
## iter  70 value 825.283859
## iter  80 value 717.283089
## iter  90 value 665.663901
## iter 100 value 662.664976
## final  value 662.664976 
## stopped after 100 iterations
## # weights:  151
## initial  value 2821.412506 
## iter  10 value 1419.093101
## final  value 1408.982681 
## converged
## # weights:  46
## initial  value 2416.380019 
## iter  10 value 1419.028012
## iter  20 value 1411.496318
## iter  30 value 1411.250820
## final  value 1411.247167 
## converged
## # weights:  67
## initial  value 1715.597492 
## iter  10 value 1417.368767
## iter  20 value 1411.417113
## iter  30 value 1411.248404
## iter  40 value 1409.858912
## iter  50 value 1409.645289
## final  value 1409.644958 
## converged
## # weights:  109
## initial  value 2312.888515 
## iter  10 value 1414.336647
## iter  20 value 1409.407812
## iter  30 value 1409.319823
## iter  40 value 1409.299754
## iter  50 value 1409.163957
## iter  60 value 1283.076438
## iter  70 value 1234.110419
## iter  80 value 999.146136
## iter  90 value 882.054324
## iter 100 value 794.244523
## final  value 794.244523 
## stopped after 100 iterations
## # weights:  151
## initial  value 2803.998688 
## iter  10 value 1422.983779
## iter  20 value 1356.092931
## iter  30 value 1193.561394
## iter  40 value 831.792869
## iter  50 value 783.063072
## iter  60 value 764.528941
## iter  70 value 738.052647
## iter  80 value 710.889011
## iter  90 value 709.124307
## iter 100 value 704.100869
## final  value 704.100869 
## stopped after 100 iterations
## # weights:  46
## initial  value 2258.495038 
## iter  10 value 1431.015938
## iter  20 value 1419.949993
## iter  30 value 1312.413572
## iter  40 value 1228.200927
## iter  50 value 1044.392163
## iter  60 value 850.450986
## iter  70 value 818.337974
## iter  80 value 791.570314
## iter  90 value 784.207982
## iter 100 value 763.710726
## final  value 763.710726 
## stopped after 100 iterations
## # weights:  67
## initial  value 2702.192678 
## iter  10 value 1423.340425
## iter  20 value 1412.863839
## iter  30 value 1412.479173
## iter  40 value 1412.436124
## iter  50 value 1390.835481
## iter  60 value 1368.240813
## iter  70 value 1190.663287
## iter  80 value 1121.465931
## iter  90 value 1083.907508
## iter 100 value 827.274825
## final  value 827.274825 
## stopped after 100 iterations
## # weights:  109
## initial  value 2597.462562 
## iter  10 value 1423.744886
## iter  20 value 1401.366982
## iter  30 value 1258.217610
## iter  40 value 1155.094128
## iter  50 value 980.708767
## iter  60 value 871.975542
## iter  70 value 822.770209
## iter  80 value 796.233453
## iter  90 value 783.479191
## iter 100 value 781.569369
## final  value 781.569369 
## stopped after 100 iterations
## # weights:  151
## initial  value 2877.548634 
## iter  10 value 1421.115682
## iter  20 value 1418.811898
## iter  30 value 1418.434061
## iter  40 value 1352.462071
## iter  50 value 1191.303792
## iter  60 value 852.399171
## iter  70 value 808.933550
## iter  80 value 804.427499
## iter  90 value 793.229162
## iter 100 value 776.909736
## final  value 776.909736 
## stopped after 100 iterations
## # weights:  109
## initial  value 4567.744758 
## iter  10 value 2108.539681
## iter  20 value 2107.861885
## final  value 2107.847391 
## converged
DryBean_TDA_KDE_5.50.5_n4_NN1Fit0
## Neural Network 
## 
## 2024 samples
##   16 predictor
##    4 classes: 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1349, 1349, 1350 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   2     0.3    0.7746947  0.6027428
##   2     0.5    0.6876638  0.4069571
##   2     0.7    0.6176591  0.2870128
##   3     0.3    0.6111206  0.2216656
##   3     0.5    0.6935898  0.4129982
##   3     0.7    0.6779045  0.3723171
##   5     0.3    0.7875569  0.6352066
##   5     0.5    0.6714760  0.3597457
##   5     0.7    0.7771653  0.6059644
##   7     0.3    0.5197626  0.0000000
##   7     0.5    0.7830904  0.6160797
##   7     0.7    0.6769161  0.3713985
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 5 and decay = 0.3.
DryBean_TDA_KDE_5.50.5_n4_NN1Fit0$resample
##    Accuracy     Kappa Resample
## 1 0.8059259 0.6830327    Fold1
## 2 0.8026706 0.6711532    Fold3
## 3 0.7540741 0.5514340    Fold2
nb_tda_kde_5.50.5_n4_nn1_fit_re<-DryBean_TDA_KDE_5.50.5_n4_NN1Fit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n4_NN1Fit0)
## a 16-5-4 network with 109 weights
## options were - softmax modelling  decay=0.3
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##    0.00    0.08    0.00    0.00    0.00    0.00    0.00    0.08    0.00    0.00 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h4  i1->h4  i2->h4  i3->h4  i4->h4  i5->h4  i6->h4  i7->h4  i8->h4  i9->h4 
##    0.00    0.02    0.00    0.00    0.00    0.00    0.00    0.02    0.00    0.00 
## i10->h4 i11->h4 i12->h4 i13->h4 i14->h4 i15->h4 i16->h4 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h5  i1->h5  i2->h5  i3->h5  i4->h5  i5->h5  i6->h5  i7->h5  i8->h5  i9->h5 
##    0.00    0.01    0.00    0.00    0.00    0.00    0.00    0.01    0.00    0.00 
## i10->h5 i11->h5 i12->h5 i13->h5 i14->h5 i15->h5 i16->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##  b->o1 h1->o1 h2->o1 h3->o1 h4->o1 h5->o1 
##   0.27   0.27   0.27   0.27   0.27   0.27 
##  b->o2 h1->o2 h2->o2 h3->o2 h4->o2 h5->o2 
##  -0.54  -0.54  -0.54  -0.54  -0.54  -0.54 
##  b->o3 h1->o3 h2->o3 h3->o3 h4->o3 h5->o3 
##   0.16   0.16   0.16   0.16   0.16   0.16 
##  b->o4 h1->o4 h2->o4 h3->o4 h4->o4 h5->o4 
##   0.11   0.11   0.11   0.11   0.11   0.11
#vip(DryBean_TDA_KDE_5.50.5_n4_NN1Fit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n4_NN1Fit TDA-Assited NN")
 

# Predict outcome using DryBean_TDA_KDE_5.50.5_n4_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n4_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n4_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n4_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      396    156  489     1063   578   608  790
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.2605          
##                  95% CI : (0.2471, 0.2743)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 0.506           
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          1.0000
## Specificity                  1.00000       1.00000      1.0000          0.0000
## Pos Pred Value                   NaN           NaN         NaN          0.2605
## Neg Pred Value               0.90294       0.96176      0.8801             NaN
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2605
## Detection Prevalence         0.00000       0.00000      0.0000          1.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
nb_tda_kde_5.50.5_n4_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON      396    156  489     1063   578   608  790
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.2605          
##                  95% CI : (0.2471, 0.2743)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 0.506           
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          1.0000
## Specificity                  1.00000       1.00000      1.0000          0.0000
## Pos Pred Value                   NaN           NaN         NaN          0.2605
## Neg Pred Value               0.90294       0.96176      0.8801             NaN
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2605
## Detection Prevalence         0.00000       0.00000      0.0000          1.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
nb_tda_kde_5.50.5_n4_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.2605392      0.0000000      0.2471237      0.2742985      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.5059787            NaN
nb_tda_kde_5.50.5_n4_db_nn1_cf0_ov_acc<-nb_tda_kde_5.50.5_n4_db_nn1_cf0$overall[1]
nb_tda_kde_5.50.5_n4_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA           0           1            NaN      0.9029412        NA
## Class: BOMBAY             0           1            NaN      0.9617647        NA
## Class: CALI               0           1            NaN      0.8801471        NA
## Class: DERMASON           1           0      0.2605392            NaN 0.2605392
## Class: HOROZ              0           1            NaN      0.8583333        NA
## Class: SEKER              0           1            NaN      0.8509804        NA
## Class: SIRA               0           1            NaN      0.8063725        NA
##                 Recall        F1 Prevalence Detection Rate Detection Prevalence
## Class: BARBUNYA      0        NA 0.09705882      0.0000000                    0
## Class: BOMBAY        0        NA 0.03823529      0.0000000                    0
## Class: CALI          0        NA 0.11985294      0.0000000                    0
## Class: DERMASON      1 0.4133774 0.26053922      0.2605392                    1
## Class: HOROZ         0        NA 0.14166667      0.0000000                    0
## Class: SEKER         0        NA 0.14901961      0.0000000                    0
## Class: SIRA          0        NA 0.19362745      0.0000000                    0
##                 Balanced Accuracy
## Class: BARBUNYA               0.5
## Class: BOMBAY                 0.5
## Class: CALI                   0.5
## Class: DERMASON               0.5
## Class: HOROZ                  0.5
## Class: SEKER                  0.5
## Class: SIRA                   0.5
nb_tda_kde_5.50.5_n4_db_nn1_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n4_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold<-(db_nn1_fit_re - nb_tda_kde_5.50.5_n4_nn1_fit_re)
diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold
##     Accuracy
## 1 -0.1003214
## 2 -0.2694632
## 3 -0.1867361
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n4_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n4_3_fold_odds.left<-bst_tda_kde_5.50.5_nn1.n4_3_fold$probLeft/bst_tda_kde_5.50.5_nn1.n4_3_fold$probRight
bst_tda_kde_5.50.5_nn1.n4_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n4_3_fold
## $winLeft
## [1] 0.9910667
## 
## $winRope
## [1] 0.008933333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n4_3_fold
## $left
## [1] 0.9552178
## 
## $rope
## [1] 0.007754273
## 
## $right
## [1] 0.03702797
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold))
#bf_tda_kde_5.50.5_nn1.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n4_3_fold)
## t = -3.799, df = 2, p-value = 0.06283
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.3956093  0.0245955
## sample estimates:
##  mean of x 
## -0.1855069
### Test set diff
diff_drybean_tda_kde_5.50.5_nn1.n4_test<-(db_nn1_cf_ov_acc - nb_tda_kde_5.50.5_n4_db_nn1_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_nn1.n4_test
##  Accuracy 
## 0.3492647
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n4_test),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n4_test_odds.left<-bst_tda_kde_5.50.5_nn1.n4_test$probLeft/bst_tda_kde_5.50.5_nn1.n4_test$probRight
bst_tda_kde_5.50.5_nn1.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n4_test),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1568333
## 
## $winRight
## [1] 0.8431667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n4_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nn1.n4_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n4_test)) #bf_tda_pca_5.50.5_nn1.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n4_test))

##Node5

#Neural Network 1

DryBean_TDA_KDE_5.50.5_n5_NN1Fit0 <- train(as.factor(Class) ~ ., data = tda.m_kde_dry_bean_dataset_5.50.5.n5.vec, 
                           Importance = T,
                     method = 'nnet', 
                     trControl = fitControl,
                     tuneGrid = nn1Grid,
                     metric='Accuracy')
## # weights:  46
## initial  value 1083.296816 
## iter  10 value 661.912163
## iter  20 value 659.703185
## iter  30 value 659.692990
## final  value 659.692912 
## converged
## # weights:  67
## initial  value 1146.084052 
## iter  10 value 662.240869
## iter  20 value 661.768047
## iter  30 value 648.269821
## iter  40 value 542.499740
## iter  50 value 458.071769
## iter  60 value 453.643266
## iter  70 value 448.041110
## iter  80 value 428.229776
## iter  90 value 414.793091
## iter 100 value 393.881233
## final  value 393.881233 
## stopped after 100 iterations
## # weights:  109
## initial  value 1966.706792 
## iter  10 value 661.431775
## iter  20 value 658.276992
## iter  30 value 658.056934
## iter  40 value 654.202860
## iter  50 value 653.137195
## iter  60 value 555.507407
## iter  70 value 514.680066
## iter  80 value 492.442697
## iter  90 value 469.866079
## iter 100 value 456.829561
## final  value 456.829561 
## stopped after 100 iterations
## # weights:  151
## initial  value 1136.840272 
## iter  10 value 666.992771
## iter  20 value 658.655091
## iter  30 value 658.580966
## iter  40 value 652.910083
## iter  50 value 638.994397
## iter  60 value 638.255321
## iter  70 value 627.910576
## iter  80 value 539.451702
## iter  90 value 488.393781
## iter 100 value 445.926432
## final  value 445.926432 
## stopped after 100 iterations
## # weights:  46
## initial  value 768.596848 
## iter  10 value 661.889019
## iter  20 value 661.739697
## iter  30 value 661.346234
## iter  40 value 564.722647
## iter  50 value 502.938420
## iter  60 value 465.920408
## iter  70 value 452.267574
## iter  80 value 417.144812
## iter  90 value 406.071972
## iter 100 value 405.976515
## final  value 405.976515 
## stopped after 100 iterations
## # weights:  67
## initial  value 942.580317 
## iter  10 value 662.295607
## iter  20 value 658.018628
## iter  30 value 602.492044
## iter  40 value 546.304489
## iter  50 value 448.544339
## iter  60 value 436.477101
## iter  70 value 434.923842
## iter  80 value 434.889023
## iter  90 value 434.873973
## iter 100 value 433.594921
## final  value 433.594921 
## stopped after 100 iterations
## # weights:  109
## initial  value 1259.519784 
## iter  10 value 681.933409
## iter  20 value 659.537516
## iter  30 value 554.932248
## iter  40 value 512.844560
## iter  50 value 480.426645
## iter  60 value 452.631911
## iter  70 value 426.178520
## iter  80 value 421.129940
## iter  90 value 412.956881
## iter 100 value 408.866546
## final  value 408.866546 
## stopped after 100 iterations
## # weights:  151
## initial  value 1090.278037 
## iter  10 value 663.554003
## iter  20 value 660.647624
## iter  30 value 659.587640
## iter  40 value 656.787749
## iter  50 value 615.408076
## iter  60 value 583.054884
## iter  70 value 544.170361
## iter  80 value 532.996225
## iter  90 value 479.269623
## iter 100 value 453.105981
## final  value 453.105981 
## stopped after 100 iterations
## # weights:  46
## initial  value 1048.359518 
## iter  10 value 672.063095
## iter  20 value 667.010432
## iter  30 value 631.704457
## iter  40 value 493.367634
## iter  50 value 450.115296
## iter  60 value 442.270164
## final  value 442.268447 
## converged
## # weights:  67
## initial  value 1078.499825 
## iter  10 value 660.405445
## iter  20 value 660.096397
## final  value 660.093951 
## converged
## # weights:  109
## initial  value 726.426094 
## iter  10 value 664.468963
## iter  20 value 659.736790
## iter  30 value 659.535082
## iter  40 value 659.498221
## iter  50 value 659.204863
## iter  60 value 659.165219
## iter  70 value 659.142292
## final  value 659.137815 
## converged
## # weights:  151
## initial  value 689.031583 
## iter  10 value 661.244627
## iter  20 value 660.109532
## iter  30 value 660.094244
## final  value 660.094184 
## converged
## # weights:  46
## initial  value 1101.256179 
## iter  10 value 669.206853
## iter  20 value 668.582319
## iter  30 value 665.603720
## iter  40 value 613.097167
## iter  50 value 486.033338
## iter  60 value 469.502237
## iter  70 value 438.908871
## iter  80 value 426.661638
## iter  90 value 403.653933
## iter 100 value 399.225500
## final  value 399.225500 
## stopped after 100 iterations
## # weights:  67
## initial  value 1058.043949 
## iter  10 value 667.487504
## iter  20 value 666.945292
## iter  30 value 664.398733
## iter  40 value 569.432910
## iter  50 value 479.339373
## iter  60 value 470.449141
## iter  70 value 457.710397
## iter  80 value 448.203644
## iter  90 value 440.389560
## iter 100 value 432.793379
## final  value 432.793379 
## stopped after 100 iterations
## # weights:  109
## initial  value 859.130258 
## iter  10 value 666.630789
## iter  20 value 665.550652
## iter  30 value 665.506270
## iter  40 value 588.918336
## iter  50 value 499.922795
## iter  60 value 470.011610
## iter  70 value 453.394337
## iter  80 value 440.413304
## iter  90 value 438.462205
## iter 100 value 432.695209
## final  value 432.695209 
## stopped after 100 iterations
## # weights:  151
## initial  value 1279.730501 
## iter  10 value 744.135382
## iter  20 value 633.939324
## iter  30 value 624.292255
## iter  40 value 594.955618
## iter  50 value 549.804395
## iter  60 value 490.812239
## iter  70 value 444.389754
## iter  80 value 415.904658
## iter  90 value 405.740346
## iter 100 value 395.552599
## final  value 395.552599 
## stopped after 100 iterations
## # weights:  46
## initial  value 930.206589 
## iter  10 value 669.035284
## iter  20 value 667.114359
## iter  30 value 667.072866
## final  value 667.072709 
## converged
## # weights:  67
## initial  value 1115.948881 
## iter  10 value 682.904581
## iter  20 value 675.837263
## iter  30 value 667.127851
## iter  40 value 666.732975
## iter  50 value 666.582811
## iter  60 value 660.108424
## iter  70 value 650.068772
## iter  80 value 564.957284
## iter  90 value 458.072837
## iter 100 value 436.975679
## final  value 436.975679 
## stopped after 100 iterations
## # weights:  109
## initial  value 919.830648 
## iter  10 value 672.260257
## iter  20 value 666.256759
## iter  30 value 660.539098
## iter  40 value 624.441703
## iter  50 value 507.602535
## iter  60 value 464.442259
## iter  70 value 455.928778
## iter  80 value 450.389771
## iter  90 value 445.554633
## iter 100 value 440.479437
## final  value 440.479437 
## stopped after 100 iterations
## # weights:  151
## initial  value 802.791351 
## iter  10 value 668.709498
## iter  20 value 667.177925
## iter  30 value 666.995905
## iter  40 value 665.581829
## iter  50 value 665.579022
## iter  50 value 665.579020
## iter  50 value 665.579020
## final  value 665.579020 
## converged
## # weights:  46
## initial  value 1079.438837 
## iter  10 value 709.807351
## iter  20 value 697.034476
## iter  30 value 672.335215
## iter  40 value 613.099826
## iter  50 value 480.365172
## iter  60 value 465.124978
## iter  70 value 454.957704
## iter  80 value 454.437292
## final  value 454.437285 
## converged
## # weights:  67
## initial  value 1031.733228 
## iter  10 value 668.420997
## iter  20 value 667.639469
## iter  30 value 659.616962
## iter  40 value 611.999336
## iter  50 value 505.634894
## iter  60 value 497.391853
## iter  70 value 490.815418
## iter  80 value 463.127676
## iter  90 value 452.658878
## final  value 452.538368 
## converged
## # weights:  109
## initial  value 1028.916739 
## iter  10 value 670.977463
## iter  20 value 646.035332
## iter  30 value 490.566479
## iter  40 value 469.664405
## iter  50 value 444.116587
## iter  60 value 432.960306
## iter  70 value 421.285188
## iter  80 value 413.856101
## iter  90 value 408.979298
## iter 100 value 406.346043
## final  value 406.346043 
## stopped after 100 iterations
## # weights:  151
## initial  value 880.004200 
## iter  10 value 670.270854
## iter  20 value 668.744855
## iter  30 value 668.013307
## iter  40 value 656.251359
## iter  50 value 540.946737
## iter  60 value 484.306918
## iter  70 value 470.653201
## iter  80 value 464.333996
## iter  90 value 454.541151
## iter 100 value 452.225293
## final  value 452.225293 
## stopped after 100 iterations
## # weights:  46
## initial  value 964.977497 
## iter  10 value 661.133423
## iter  20 value 658.493850
## iter  30 value 658.469934
## iter  40 value 658.357033
## iter  50 value 658.343595
## final  value 658.343419 
## converged
## # weights:  67
## initial  value 1531.308266 
## iter  10 value 665.099232
## iter  20 value 663.892976
## iter  30 value 658.027215
## iter  40 value 581.959989
## iter  50 value 511.586323
## iter  60 value 453.642052
## iter  70 value 438.350853
## iter  80 value 418.216801
## iter  90 value 412.280640
## iter 100 value 410.917889
## final  value 410.917889 
## stopped after 100 iterations
## # weights:  109
## initial  value 1263.796691 
## iter  10 value 657.944481
## iter  20 value 657.878887
## iter  30 value 657.703826
## iter  40 value 627.861482
## iter  50 value 593.475602
## iter  60 value 510.833699
## iter  70 value 444.160047
## iter  80 value 417.036336
## iter  90 value 412.414392
## iter 100 value 410.587902
## final  value 410.587902 
## stopped after 100 iterations
## # weights:  151
## initial  value 1024.493836 
## iter  10 value 667.603355
## iter  20 value 655.438431
## iter  30 value 450.002509
## iter  40 value 429.874600
## iter  50 value 421.560761
## iter  60 value 407.871646
## iter  70 value 373.359127
## iter  80 value 354.123562
## iter  90 value 347.193016
## iter 100 value 346.321688
## final  value 346.321688 
## stopped after 100 iterations
## # weights:  46
## initial  value 1081.310390 
## iter  10 value 671.295674
## iter  20 value 661.301281
## iter  30 value 660.734577
## iter  40 value 659.670961
## iter  50 value 659.455845
## final  value 659.454591 
## converged
## # weights:  67
## initial  value 1390.307914 
## iter  10 value 665.826474
## iter  20 value 659.254738
## iter  30 value 658.773901
## iter  40 value 658.772251
## final  value 658.772238 
## converged
## # weights:  109
## initial  value 1036.110575 
## iter  10 value 686.208639
## iter  20 value 660.414545
## iter  30 value 659.988932
## iter  30 value 659.988929
## iter  40 value 659.904196
## iter  50 value 637.190520
## iter  60 value 522.979497
## iter  70 value 432.146770
## iter  80 value 413.710994
## iter  90 value 393.595965
## iter 100 value 381.346541
## final  value 381.346541 
## stopped after 100 iterations
## # weights:  151
## initial  value 1029.580609 
## iter  10 value 661.439377
## iter  20 value 658.795407
## iter  30 value 657.784758
## iter  40 value 657.675580
## iter  50 value 657.667491
## final  value 657.667450 
## converged
## # weights:  46
## initial  value 942.551047 
## iter  10 value 670.725087
## iter  20 value 663.084824
## iter  30 value 662.375116
## iter  40 value 643.814142
## iter  50 value 494.879507
## iter  60 value 457.661155
## iter  70 value 441.166777
## iter  80 value 429.146651
## iter  90 value 427.077835
## iter 100 value 426.937268
## final  value 426.937268 
## stopped after 100 iterations
## # weights:  67
## initial  value 755.608832 
## iter  10 value 665.442429
## iter  20 value 662.194372
## iter  30 value 662.111778
## iter  40 value 661.577413
## iter  50 value 660.066882
## iter  60 value 659.842260
## iter  70 value 659.707834
## iter  80 value 659.594713
## final  value 659.586819 
## converged
## # weights:  109
## initial  value 1042.759471 
## iter  10 value 660.523377
## iter  20 value 659.150862
## iter  30 value 658.793302
## iter  40 value 658.738221
## iter  50 value 658.674761
## iter  60 value 658.320896
## iter  70 value 645.851991
## iter  80 value 465.751673
## iter  90 value 447.258280
## iter 100 value 433.849439
## final  value 433.849439 
## stopped after 100 iterations
## # weights:  151
## initial  value 1116.443093 
## iter  10 value 659.043634
## iter  20 value 658.605344
## iter  30 value 658.417304
## iter  40 value 658.345445
## iter  50 value 454.832492
## iter  60 value 429.357577
## iter  70 value 415.860148
## iter  80 value 405.151339
## iter  90 value 390.634884
## iter 100 value 374.107387
## final  value 374.107387 
## stopped after 100 iterations
## # weights:  151
## initial  value 2253.814575 
## iter  10 value 1008.855893
## iter  20 value 992.496092
## iter  30 value 990.592991
## iter  40 value 944.778643
## iter  50 value 923.802498
## iter  60 value 904.121902
## iter  70 value 793.454004
## iter  80 value 676.678024
## iter  90 value 667.323414
## iter 100 value 645.657470
## final  value 645.657470 
## stopped after 100 iterations
DryBean_TDA_KDE_5.50.5_n5_NN1Fit0
## Neural Network 
## 
## 989 samples
##  16 predictor
##   4 classes: 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 659, 661, 658 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   2     0.3    0.6251961  0.1827544
##   2     0.5    0.6187537  0.1755853
##   2     0.7    0.7240180  0.4723262
##   3     0.3    0.7260290  0.4942634
##   3     0.5    0.6695728  0.3190770
##   3     0.7    0.6211311  0.1679244
##   5     0.3    0.7229956  0.4706652
##   5     0.5    0.7219978  0.4694074
##   5     0.7    0.6684624  0.3086276
##   7     0.3    0.7402044  0.5259517
##   7     0.5    0.6177436  0.1604981
##   7     0.7    0.6684532  0.3100571
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 7 and decay = 0.3.
DryBean_TDA_KDE_5.50.5_n5_NN1Fit0$resample
##    Accuracy     Kappa Resample
## 1 0.7652439 0.5754152    Fold2
## 2 0.7272727 0.4814944    Fold1
## 3 0.7280967 0.5209456    Fold3
nb_tda_kde_5.50.5_n5_nn1_fit_re<-DryBean_TDA_KDE_5.50.5_n5_NN1Fit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n5_NN1Fit0)
## a 16-7-4 network with 151 weights
## options were - softmax modelling  decay=0.3
##   b->h1  i1->h1  i2->h1  i3->h1  i4->h1  i5->h1  i6->h1  i7->h1  i8->h1  i9->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h1 i11->h1 i12->h1 i13->h1 i14->h1 i15->h1 i16->h1 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h2  i1->h2  i2->h2  i3->h2  i4->h2  i5->h2  i6->h2  i7->h2  i8->h2  i9->h2 
##    0.00    0.07    0.00    0.00    0.00    0.00    0.00    0.07    0.00    0.00 
## i10->h2 i11->h2 i12->h2 i13->h2 i14->h2 i15->h2 i16->h2 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h3  i1->h3  i2->h3  i3->h3  i4->h3  i5->h3  i6->h3  i7->h3  i8->h3  i9->h3 
##   -0.01   -0.02   -0.16    1.13    0.74   -0.02    0.00    0.01   -1.13    0.05 
## i10->h3 i11->h3 i12->h3 i13->h3 i14->h3 i15->h3 i16->h3 
##   -0.01   -0.02   -0.02    0.00    0.00   -0.02   -0.03 
##   b->h4  i1->h4  i2->h4  i3->h4  i4->h4  i5->h4  i6->h4  i7->h4  i8->h4  i9->h4 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h4 i11->h4 i12->h4 i13->h4 i14->h4 i15->h4 i16->h4 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h5  i1->h5  i2->h5  i3->h5  i4->h5  i5->h5  i6->h5  i7->h5  i8->h5  i9->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h5 i11->h5 i12->h5 i13->h5 i14->h5 i15->h5 i16->h5 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h6  i1->h6  i2->h6  i3->h6  i4->h6  i5->h6  i6->h6  i7->h6  i8->h6  i9->h6 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
## i10->h6 i11->h6 i12->h6 i13->h6 i14->h6 i15->h6 i16->h6 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##   b->h7  i1->h7  i2->h7  i3->h7  i4->h7  i5->h7  i6->h7  i7->h7  i8->h7  i9->h7 
##    0.00    0.01    0.00    0.00    0.00    0.00    0.00    0.01    0.00    0.00 
## i10->h7 i11->h7 i12->h7 i13->h7 i14->h7 i15->h7 i16->h7 
##    0.00    0.00    0.00    0.00    0.00    0.00    0.00 
##  b->o1 h1->o1 h2->o1 h3->o1 h4->o1 h5->o1 h6->o1 h7->o1 
##   0.12   0.12   0.12   2.27   0.17   0.00   0.12   0.11 
##  b->o2 h1->o2 h2->o2 h3->o2 h4->o2 h5->o2 h6->o2 h7->o2 
##  -0.62  -0.62  -0.62   0.41  -0.03   0.00  -0.62  -0.58 
##  b->o3 h1->o3 h2->o3 h3->o3 h4->o3 h5->o3 h6->o3 h7->o3 
##   0.56   0.56   0.56  -5.11  -0.19   0.00   0.56   0.53 
##  b->o4 h1->o4 h2->o4 h3->o4 h4->o4 h5->o4 h6->o4 h7->o4 
##  -0.07  -0.07  -0.07   2.43   0.05   0.00  -0.07  -0.07
#vip(DryBean_TDA_KDE_5.50.5_n5_NN1Fit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n5_NN1Fit TDA-Assited NN")


# Predict outcome using DryBean_TDA_KDE_5.50.5_n5_NN1Fit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n5_NN1Fit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n5_db_nn1_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n5_db_nn1_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON       13      0   91     1038   573    21  662
##   HOROZ           0      0    0        0     0     0    0
##   SEKER         383    156  398       25     5   587  128
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3983          
##                  95% CI : (0.3832, 0.4135)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2339          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9765
## Specificity                  1.00000       1.00000      1.0000          0.5492
## Pos Pred Value                   NaN           NaN         NaN          0.4329
## Neg Pred Value               0.90294       0.96176      0.8801          0.9851
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2544
## Detection Prevalence         0.00000       0.00000      0.0000          0.5877
## Balanced Accuracy            0.50000       0.50000      0.5000          0.7629
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000       0.9655      0.0000
## Specificity                1.0000       0.6846      1.0000
## Pos Pred Value                NaN       0.3490         NaN
## Neg Pred Value             0.8583       0.9912      0.8064
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.0000       0.1439      0.0000
## Detection Prevalence       0.0000       0.4123      0.0000
## Balanced Accuracy          0.5000       0.8250      0.5000
nb_tda_kde_5.50.5_n5_db_nn1_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON       13      0   91     1038   573    21  662
##   HOROZ           0      0    0        0     0     0    0
##   SEKER         383    156  398       25     5   587  128
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3983          
##                  95% CI : (0.3832, 0.4135)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2339          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9765
## Specificity                  1.00000       1.00000      1.0000          0.5492
## Pos Pred Value                   NaN           NaN         NaN          0.4329
## Neg Pred Value               0.90294       0.96176      0.8801          0.9851
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2544
## Detection Prevalence         0.00000       0.00000      0.0000          0.5877
## Balanced Accuracy            0.50000       0.50000      0.5000          0.7629
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000       0.9655      0.0000
## Specificity                1.0000       0.6846      1.0000
## Pos Pred Value                NaN       0.3490         NaN
## Neg Pred Value             0.8583       0.9912      0.8064
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.0000       0.1439      0.0000
## Detection Prevalence       0.0000       0.4123      0.0000
## Balanced Accuracy          0.5000       0.8250      0.5000
nb_tda_kde_5.50.5_n5_db_nn1_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   3.982843e-01   2.339080e-01   3.832177e-01   4.134967e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##   4.962694e-82            NaN
nb_tda_kde_5.50.5_n5_db_nn1_cf0_ov_acc<-nb_tda_kde_5.50.5_n5_db_nn1_cf0$overall[1]
nb_tda_kde_5.50.5_n5_db_nn1_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.0000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.0000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON   0.9764817   0.5492211      0.4328607      0.9851367 0.4328607
## Class: HOROZ      0.0000000   1.0000000            NaN      0.8583333        NA
## Class: SEKER      0.9654605   0.6846198      0.3489893      0.9912427 0.3489893
## Class: SIRA       0.0000000   1.0000000            NaN      0.8063725        NA
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.0000000        NA 0.09705882      0.0000000
## Class: BOMBAY   0.0000000        NA 0.03823529      0.0000000
## Class: CALI     0.0000000        NA 0.11985294      0.0000000
## Class: DERMASON 0.9764817 0.5998266 0.26053922      0.2544118
## Class: HOROZ    0.0000000        NA 0.14166667      0.0000000
## Class: SEKER    0.9654605 0.5126638 0.14901961      0.1438725
## Class: SIRA     0.0000000        NA 0.19362745      0.0000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA            0.0000000         0.5000000
## Class: BOMBAY              0.0000000         0.5000000
## Class: CALI                0.0000000         0.5000000
## Class: DERMASON            0.5877451         0.7628514
## Class: HOROZ               0.0000000         0.5000000
## Class: SEKER               0.4122549         0.8250402
## Class: SIRA                0.0000000         0.5000000
nb_tda_kde_5.50.5_n5_db_nn1_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n5_db_nn1_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold<-(db_nn1_fit_re - nb_tda_kde_5.50.5_n5_nn1_fit_re)
diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold
##      Accuracy
## 1 -0.05963937
## 2 -0.19406530
## 3 -0.16075873
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n5_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n5_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n5_3_fold_odds.left<-bst_tda_kde_5.50.5_nn1.n5_3_fold$probLeft/bst_tda_kde_5.50.5_nn1.n5_3_fold$probRight
bst_tda_kde_5.50.5_nn1.n5_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n5_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n5_3_fold
## $winLeft
## [1] 0.9920667
## 
## $winRope
## [1] 0.007933333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n5_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n5_3_fold
## $left
## [1] 0.9445112
## 
## $rope
## [1] 0.01221637
## 
## $right
## [1] 0.0432724
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold))
#bf_tda_kde_5.50.5_nn1.n5_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_nn1_n5_3_fold)
## t = -3.4182, df = 2, p-value = 0.07596
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.31205825  0.03574932
## sample estimates:
##  mean of x 
## -0.1381545
### Test set diff
diff_drybean_tda_kde_5.50.5_nn1.n5_test<-(db_nn1_cf_ov_acc - nb_tda_kde_5.50.5_n5_db_nn1_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_nn1.n5_test
##  Accuracy 
## 0.2115196
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nn1.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n5_test),-0.01,0.01)
bst_tda_kde_5.50.5_nn1.n5_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nn1.n5_test_odds.left<-bst_tda_kde_5.50.5_nn1.n5_test$probLeft/bst_tda_kde_5.50.5_nn1.n5_test$probRight
bst_tda_kde_5.50.5_nn1.n5_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nn1.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n4_test),-0.01,0.01)
bsr_tda_kde_5.50.5_nn1.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1615667
## 
## $winRight
## [1] 0.8384333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nn1.n5_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n5_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nn1.n5_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nn1.n5_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nn1.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n5_test)) #bf_tda_pca_5.50.5_nn1.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nn1.n5_test)) 


##Logistic Regression  method='multinom'

dryBeanLrFit <- train(as.factor(Class) ~ ., 
                 data = Dry_Bean_DatasetTrain, 
                 family = 'binomial',
                method = 'multinom', 
                 trControl = fitControl,
                metric='Accuracy')
## # weights:  126 (102 variable)
## initial  value 12362.367177 
## iter  10 value 9330.369345
## iter  20 value 6749.354731
## iter  30 value 5678.621798
## iter  40 value 2359.286993
## iter  50 value 1381.324503
## iter  60 value 1303.958222
## iter  70 value 1280.976922
## iter  80 value 1261.906693
## iter  90 value 1252.052252
## iter 100 value 1243.031033
## final  value 1243.031033 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12362.367177 
## iter  10 value 9330.369392
## iter  20 value 6749.356862
## iter  30 value 5678.995387
## iter  40 value 2512.636380
## iter  50 value 1725.772824
## iter  60 value 1610.816489
## iter  70 value 1531.708206
## iter  80 value 1482.938511
## iter  90 value 1465.564607
## iter 100 value 1452.820198
## final  value 1452.820198 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12362.367177 
## iter  10 value 9330.369345
## iter  20 value 6749.354729
## iter  30 value 5678.623940
## iter  40 value 2360.172572
## iter  50 value 1382.779074
## iter  60 value 1307.338376
## iter  70 value 1285.935476
## iter  80 value 1269.615240
## iter  90 value 1262.230024
## iter 100 value 1255.496272
## final  value 1255.496272 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12366.258997 
## iter  10 value 9324.925181
## iter  20 value 7332.973845
## iter  30 value 5126.001635
## iter  40 value 2638.877179
## iter  50 value 1423.772588
## iter  60 value 1345.259517
## iter  70 value 1328.387799
## iter  80 value 1310.396748
## iter  90 value 1298.268238
## iter 100 value 1291.770919
## final  value 1291.770919 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12366.258997 
## iter  10 value 9324.925219
## iter  20 value 7332.975459
## iter  30 value 5126.608164
## iter  40 value 2722.252546
## iter  50 value 1777.540654
## iter  60 value 1655.443575
## iter  70 value 1580.415537
## iter  80 value 1528.144994
## iter  90 value 1515.229401
## iter 100 value 1504.569564
## final  value 1504.569564 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12366.258997 
## iter  10 value 9324.925181
## iter  20 value 7332.973850
## iter  30 value 5126.002046
## iter  40 value 2638.937341
## iter  50 value 1425.421293
## iter  60 value 1348.924858
## iter  70 value 1333.437606
## iter  80 value 1318.416578
## iter  90 value 1309.319649
## iter 100 value 1304.157066
## final  value 1304.157066 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12364.313087 
## iter  10 value 9303.251690
## iter  20 value 6747.215042
## iter  30 value 5136.956724
## iter  40 value 2710.122161
## iter  50 value 1365.293349
## iter  60 value 1299.474377
## iter  70 value 1281.985811
## iter  80 value 1267.371275
## iter  90 value 1258.602573
## iter 100 value 1245.996684
## final  value 1245.996684 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12364.313087 
## iter  10 value 9303.251723
## iter  20 value 6747.217164
## iter  30 value 5137.231414
## iter  40 value 2681.655242
## iter  50 value 1698.885165
## iter  60 value 1576.613279
## iter  70 value 1501.036571
## iter  80 value 1440.450343
## iter  90 value 1424.106938
## iter 100 value 1414.444134
## final  value 1414.444134 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12364.313087 
## iter  10 value 9303.251690
## iter  20 value 6747.215042
## iter  30 value 5136.956830
## iter  40 value 2709.853870
## iter  50 value 1366.718618
## iter  60 value 1302.274050
## iter  70 value 1285.880948
## iter  80 value 1273.350491
## iter  90 value 1266.768135
## iter 100 value 1257.838280
## final  value 1257.838280 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 18546.469631 
## iter  10 value 14256.077488
## iter  20 value 10801.375725
## iter  30 value 8317.981350
## iter  40 value 4842.299633
## iter  50 value 2094.588199
## iter  60 value 1992.650958
## iter  70 value 1957.386666
## iter  80 value 1940.152053
## iter  90 value 1927.837065
## iter 100 value 1919.271883
## final  value 1919.271883 
## stopped after 100 iterations
dryBeanLrFit
## Penalized Multinomial Regression 
## 
## 9531 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6353, 6355, 6354 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.9264521  0.9110561
##   1e-04  0.9268719  0.9115603
##   1e-01  0.9224645  0.9062325
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 1e-04.
dryBeanLrFit$resample
##    Accuracy     Kappa Resample
## 1 0.9363980 0.9230796    Fold2
## 2 0.9203902 0.9037874    Fold1
## 3 0.9238275 0.9078139    Fold3
db_lr_fit_re<-dryBeanLrFit$resample[1]

summary(dryBeanLrFit)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##          (Intercept)        Area   Perimeter MajorAxisLength MinorAxisLength
## BOMBAY      5.097996 0.001518203 -0.08485485       0.6867606        1.444120
## CALI       27.758816 0.003852931 -0.18871191       2.1712986        2.872933
## DERMASON   22.581674 0.005414543  0.19965082       1.1314684        1.626920
## HOROZ       4.712840 0.008208654  0.09969379       2.5184362        4.258753
## SEKER      -8.485302 0.008216020  0.15925215      -0.2937576       -1.380356
## SIRA       66.354878 0.004679263 -0.38854574       2.2510720        2.947044
##          AspectRation Eccentricity    ConvexArea EquivDiameter     Extent
## BOMBAY       37.28312     7.644902 -9.677267e-06    -2.4349872  -8.321374
## CALI        -60.69857    95.595706 -3.981068e-03    -4.2920059   4.056257
## DERMASON    -21.95784    74.657652 -3.629310e-03    -4.4236490 -16.330867
## HOROZ       -16.65023    97.110516 -7.241419e-03    -7.4567788  -6.170527
## SEKER       -22.19845   -48.301470 -7.531226e-03     0.5973303 -11.721895
## SIRA        -55.06946   134.255959 -4.635402e-03    -4.0925410  -6.884650
##           Solidity  roundness Compactness ShapeFactor1 ShapeFactor2
## BOMBAY    7.968895   17.74026    1.854190   0.67565651   0.18278340
## CALI     28.500269  -46.46146    5.614737   0.71126202   0.13176749
## DERMASON 11.057829  139.34613    9.193598   0.27073145   0.01648839
## HOROZ    41.195855   74.66557  -25.228524   0.88101195  -0.21155705
## SEKER    -7.785538   97.57366   25.634165  -1.11510060   0.06150360
## SIRA     42.793980 -144.76055   42.479503   0.06103648  -0.02510808
##          ShapeFactor3 ShapeFactor4
## BOMBAY       1.028511     8.450051
## CALI       -23.183451    -3.648962
## DERMASON   -13.521871     5.847190
## HOROZ      -58.066290    -4.029509
## SEKER       59.277237     8.638673
## SIRA         3.867839    28.032780
## 
## Std. Errors:
##           (Intercept)         Area   Perimeter MajorAxisLength MinorAxisLength
## BOMBAY   7.917461e-06 0.0026703592 0.004074527    0.0015121523    0.0009994635
## CALI     2.831263e-06 0.0003613638 0.001249474    0.0006797879    0.0005053984
## DERMASON 7.794527e-06 0.0007816329 0.002488559    0.0017441752    0.0019341325
## HOROZ    4.011739e-06 0.0004717857 0.001717104    0.0005561916    0.0006173356
## SEKER    4.745019e-06 0.0009470733 0.002047941    0.0005773709    0.0006787073
## SIRA     6.308116e-06 0.0005622078 0.002006143    0.0023566795    0.0024504302
##          AspectRation Eccentricity   ConvexArea EquivDiameter       Extent
## BOMBAY   1.234439e-05 5.872410e-06 0.0026328242  0.0012264358 5.719707e-06
## CALI     6.003313e-06 2.287693e-06 0.0003583472  0.0003895703 2.388238e-06
## DERMASON 1.956993e-05 7.462169e-06 0.0007898878  0.0008631003 7.606134e-06
## HOROZ    5.134695e-06 2.750474e-06 0.0004665014  0.0005727688 3.068941e-06
## SEKER    5.117149e-06 2.835002e-06 0.0009451085  0.0006541179 3.776227e-06
## SIRA     2.518209e-05 8.250341e-06 0.0005627223  0.0007952691 6.852901e-06
##              Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## BOMBAY   7.820727e-06 7.027581e-06 6.393240e-06 6.217948e-08 1.621544e-08
## CALI     2.792485e-06 2.793458e-06 2.751830e-06 2.488082e-08 9.092922e-09
## DERMASON 7.706436e-06 9.218825e-06 1.148968e-05 6.718853e-08 6.240651e-08
## HOROZ    3.945310e-06 3.816346e-06 3.693618e-06 3.169838e-08 1.114481e-08
## SEKER    4.698978e-06 4.455297e-06 4.407947e-06 3.727738e-08 1.494083e-08
## SIRA     6.209524e-06 9.457196e-06 1.267679e-05 3.983808e-08 6.754685e-08
##          ShapeFactor3 ShapeFactor4
## BOMBAY   5.192961e-06 7.912661e-06
## CALI     2.787191e-06 2.819754e-06
## DERMASON 1.428358e-05 7.804055e-06
## HOROZ    3.351762e-06 3.985891e-06
## SEKER    4.034923e-06 4.740540e-06
## SIRA     1.667630e-05 6.386398e-06
## 
## Residual Deviance: 3838.544 
## AIC: 4042.544
vip(dryBeanLrFit,25) + ggtitle('non-TDA-Assisted LR')

# Predict outcome using model from training data based on testing data
predictions <- predict(dryBeanLrFit, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_lr_cf<-confusionMatrix(data=predictions, as.factor(Dry_Bean_DatasetTest$Class))
db_lr_cf
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   17        0     1     8    3
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           20      0  454        0    11     1    1
##   DERMASON        0      0    0      954     6    10   71
##   HOROZ           2      0   11        2   552     0   11
##   SEKER           2      0    1       21     0   572    2
##   SIRA            8      0    6       86     8    17  702
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9201          
##                  95% CI : (0.9114, 0.9282)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.9034          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000      0.9284          0.8975
## Specificity                  0.99213       1.00000      0.9908          0.9712
## Pos Pred Value               0.92621       1.00000      0.9322          0.9164
## Neg Pred Value               0.99132       1.00000      0.9903          0.9641
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.03824      0.1113          0.2338
## Detection Prevalence         0.09632       0.03824      0.1194          0.2551
## Balanced Accuracy            0.95566       1.00000      0.9596          0.9343
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9550       0.9408      0.8886
## Specificity                0.9926       0.9925      0.9620
## Pos Pred Value             0.9550       0.9565      0.8489
## Neg Pred Value             0.9926       0.9897      0.9729
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1353       0.1402      0.1721
## Detection Prevalence       0.1417       0.1466      0.2027
## Balanced Accuracy          0.9738       0.9667      0.9253
db_lr_cf$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.9200980      0.9034124      0.9113514      0.9282373      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_lr_cf_ov_acc<-db_lr_cf$overall[1]
db_lr_cf$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9191919   0.9921281      0.9262087      0.9913209 0.9262087
## Class: BOMBAY     1.0000000   1.0000000      1.0000000      1.0000000 1.0000000
## Class: CALI       0.9284254   0.9908104      0.9322382      0.9902588 0.9322382
## Class: DERMASON   0.8974600   0.9711634      0.9164265      0.9641329 0.9164265
## Class: HOROZ      0.9550173   0.9925757      0.9550173      0.9925757 0.9550173
## Class: SEKER      0.9407895   0.9925115      0.9565217      0.9896611 0.9565217
## Class: SIRA       0.8886076   0.9620061      0.8488513      0.9729480 0.8488513
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9191919 0.9226869 0.09705882     0.08921569
## Class: BOMBAY   1.0000000 1.0000000 0.03823529     0.03823529
## Class: CALI     0.9284254 0.9303279 0.11985294     0.11127451
## Class: DERMASON 0.8974600 0.9068441 0.26053922     0.23382353
## Class: HOROZ    0.9550173 0.9550173 0.14166667     0.13529412
## Class: SEKER    0.9407895 0.9485904 0.14901961     0.14019608
## Class: SIRA     0.8886076 0.8682746 0.19362745     0.17205882
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09632353         0.9556600
## Class: BOMBAY             0.03823529         1.0000000
## Class: CALI               0.11936275         0.9596179
## Class: DERMASON           0.25514706         0.9343117
## Class: HOROZ              0.14166667         0.9737965
## Class: SEKER              0.14656863         0.9666505
## Class: SIRA               0.20269608         0.9253068
db_lr_cf_pre_rec_f1<-db_lr_cf$byClass[5:7]


##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node1

DryBean_TDA_PC_5.50.5_n1_LrFit0 <- multinom(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n1.vec, family = 'binomial')
## # weights:  108 (85 variable)
## initial  value 14045.602479 
## iter  10 value 3976.144108
## iter  20 value 3670.317727
## iter  30 value 2455.719076
## iter  40 value 1931.457008
## iter  50 value 1856.281767
## iter  60 value 1829.581353
## iter  70 value 1821.171008
## iter  80 value 1804.889762
## iter  90 value 1799.122621
## iter 100 value 1797.526628
## final  value 1797.526628 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n1_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_dry_bean_dataset_5.50.5.n1.vec, 
                    family = 'binomial',
                          method = 'multinom', 
                    trControl = fitControl,
                          metric='Accuracy')
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2443.746575
## iter  20 value 2093.043697
## iter  30 value 1587.296150
## iter  40 value 1290.380255
## iter  50 value 1245.870952
## iter  60 value 1222.553446
## iter  70 value 1217.809803
## iter  80 value 1201.856647
## iter  90 value 1195.943338
## iter 100 value 1193.414757
## final  value 1193.414757 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2443.748044
## iter  20 value 2093.055139
## iter  30 value 1623.256265
## iter  40 value 1332.519927
## iter  50 value 1319.578901
## iter  60 value 1319.150857
## iter  70 value 1318.996536
## final  value 1318.996118 
## converged
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2443.746576
## iter  20 value 2093.043714
## iter  30 value 1587.304683
## iter  40 value 1290.640806
## iter  50 value 1247.181782
## iter  60 value 1225.794821
## iter  70 value 1221.655723
## iter  80 value 1209.663244
## iter  90 value 1206.200492
## iter 100 value 1204.878911
## final  value 1204.878911 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2566.846410
## iter  20 value 2260.870551
## iter  30 value 1641.776203
## iter  40 value 1292.790518
## iter  50 value 1230.502926
## iter  60 value 1200.099350
## iter  70 value 1192.915489
## iter  80 value 1182.844068
## iter  90 value 1174.130184
## iter 100 value 1171.460590
## final  value 1171.460590 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2566.847825
## iter  20 value 2260.874011
## iter  30 value 1783.076382
## iter  40 value 1331.819176
## iter  50 value 1309.140681
## iter  60 value 1308.059692
## iter  70 value 1307.792999
## final  value 1307.785534 
## converged
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2566.846411
## iter  20 value 2260.870562
## iter  30 value 1642.689583
## iter  40 value 1295.918669
## iter  50 value 1236.654500
## iter  60 value 1205.186961
## iter  70 value 1197.730223
## iter  80 value 1189.876552
## iter  90 value 1184.099164
## iter 100 value 1182.278289
## final  value 1182.278289 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2458.954630
## iter  20 value 2072.499864
## iter  30 value 1409.906716
## iter  40 value 1277.434269
## iter  50 value 1232.353128
## iter  60 value 1219.141190
## iter  70 value 1215.590912
## iter  80 value 1209.665427
## iter  90 value 1204.289742
## iter 100 value 1201.576611
## final  value 1201.576611 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2458.956084
## iter  20 value 2072.511111
## iter  30 value 1435.173320
## iter  40 value 1333.168473
## iter  50 value 1329.555932
## iter  60 value 1328.989434
## iter  70 value 1328.898097
## final  value 1328.898001 
## converged
## # weights:  108 (85 variable)
## initial  value 9363.734986 
## iter  10 value 2458.954632
## iter  20 value 2072.499886
## iter  30 value 1409.952236
## iter  40 value 1277.841337
## iter  50 value 1234.546414
## iter  60 value 1222.545718
## iter  70 value 1219.467621
## iter  80 value 1215.109436
## iter  90 value 1211.586692
## iter 100 value 1210.299762
## final  value 1210.299762 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 14045.602479 
## iter  10 value 3976.144108
## iter  20 value 3670.317727
## iter  30 value 2455.719076
## iter  40 value 1931.457008
## iter  50 value 1856.281767
## iter  60 value 1829.581353
## iter  70 value 1821.171008
## iter  80 value 1804.889762
## iter  90 value 1799.122621
## iter 100 value 1797.526628
## final  value 1797.526628 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n1_LrFit0
## Penalized Multinomial Regression 
## 
## 7839 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5226, 5226, 5226 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.9100651  0.8614578
##   1e-04  0.9095548  0.8606863
##   1e-01  0.9057278  0.8546027
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 0.
DryBean_TDA_PC_5.50.5_n1_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9054726 0.8540440    Fold3
## 2 0.9100651 0.8616784    Fold2
## 3 0.9146575 0.8686511    Fold1
db_tda_pc_5.50.5_n1_lr_fit_re<-DryBean_TDA_PC_5.50.5_n1_LrFit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n1_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##          (Intercept)         Area    Perimeter MajorAxisLength MinorAxisLength
## CALI       -6.805172  0.014524813 -0.002344032      -3.6305275       -5.106682
## DERMASON   21.142337  0.003319246  0.214427739       1.4746151        1.826362
## HOROZ     -11.413209 -0.001031752 -0.174726641       1.4669020        2.989412
## SEKER     -33.635268  0.007845274  0.296875870      -0.2400706       -2.167316
## SIRA       54.976640  0.002473839 -0.365733129       2.7140917        3.183017
##          AspectRation Eccentricity   ConvexArea EquivDiameter    Extent
## CALI        0.2363331    -16.05648 -0.016098460      9.296873 -43.48817
## DERMASON  -14.4931013     27.19918 -0.002846655     -4.524010 -34.52531
## HOROZ      17.7647076     49.15071 -0.003389686     -2.387465 -31.79814
## SEKER     -39.1032462    -97.33689 -0.007449825      0.928775 -30.27408
## SIRA      -70.2706885    115.31302 -0.002561091     -4.814693 -25.62583
##            Solidity  roundness Compactness ShapeFactor1 ShapeFactor2
## CALI      -6.713114  -18.33349   -4.193017  -0.11033846  -0.01564704
## DERMASON -19.211617  141.63227   11.189794   0.74861814   0.44729411
## HOROZ     -9.718945  -29.79388  -39.588675   0.07322316  -0.47407662
## SEKER    -18.943963  150.17355   30.755231  -1.47096929   0.13169631
## SIRA      71.746834 -139.74119   40.366393   0.77262522   0.28817003
##          ShapeFactor3 ShapeFactor4
## CALI       -0.3147478    -4.208654
## DERMASON   -5.1900489    -3.468595
## HOROZ     -63.3798463   -24.910382
## SEKER      93.6830863     8.790040
## SIRA       10.2798160    32.025067
## 
## Std. Errors:
##           (Intercept)         Area   Perimeter MajorAxisLength MinorAxisLength
## CALI     6.242985e-07 0.0001022191 0.000297086    0.0001044604    5.629651e-05
## DERMASON 6.696232e-06 0.0009880656 0.001919046    0.0020467490    2.104018e-03
## HOROZ    6.179209e-06 0.0020296692 0.002577041    0.0012009467    3.048743e-04
## SEKER    4.570700e-06 0.0008911461 0.001977499    0.0008096503    4.195887e-04
## SIRA     5.854455e-06 0.0009315765 0.001913747    0.0021066623    2.007447e-03
##          AspectRation Eccentricity   ConvexArea EquivDiameter       Extent
## CALI     9.939293e-07 4.998684e-07 0.0001000448  7.738052e-05 4.389490e-07
## DERMASON 2.306598e-05 8.607583e-06 0.0009840845  7.250028e-04 6.745788e-06
## HOROZ    1.457193e-05 6.069319e-06 0.0020115505  6.609513e-04 4.647886e-06
## SEKER    7.488201e-06 4.002717e-06 0.0008882299  5.648664e-04 3.422807e-06
## SIRA     2.305160e-05 8.665284e-06 0.0009284368  6.707533e-04 6.100129e-06
##              Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## CALI     6.163512e-07 4.893486e-07 4.910909e-07 5.954112e-09 1.548476e-09
## DERMASON 6.617547e-06 8.796041e-06 1.180564e-05 6.230080e-08 6.752508e-08
## HOROZ    6.115574e-06 4.607653e-06 3.819712e-06 8.155191e-08 9.524541e-09
## SEKER    4.522457e-06 4.041766e-06 3.660598e-06 4.370569e-08 1.329759e-08
## SIRA     5.786258e-06 7.784422e-06 1.084717e-05 5.465057e-08 6.181417e-08
##          ShapeFactor3 ShapeFactor4
## CALI     3.842223e-07 6.232035e-07
## DERMASON 1.550949e-05 6.720790e-06
## HOROZ    2.227296e-06 6.175856e-06
## SEKER    3.041702e-06 4.557710e-06
## SIRA     1.449632e-05 5.881349e-06
## 
## Residual Deviance: 3595.053 
## AIC: 3765.053
vip(DryBean_TDA_PC_5.50.5_n1_LrFit0,50) + ggtitle("dryBean_TDA_PCA_5.50.5_n1_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_PC_5.50.5_n1_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n1_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n1_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n1_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      150     29   12        0     0     5    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0      963    31    10   75
##   HOROZ           0      0    0        1    21     0    0
##   SEKER         224    127  165       21     6   575    3
##   SIRA           22      0  312       78   520    18  712
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5934          
##                  95% CI : (0.5781, 0.6085)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.4954          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.37879       0.00000      0.0000          0.9059
## Specificity                  0.98751       1.00000      1.0000          0.9616
## Pos Pred Value               0.76531           NaN         NaN          0.8925
## Neg Pred Value               0.93666       0.96176      0.8801          0.9667
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.03676       0.00000      0.0000          0.2360
## Detection Prevalence         0.04804       0.00000      0.0000          0.2645
## Balanced Accuracy            0.68315       0.50000      0.5000          0.9337
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity              0.036332       0.9457      0.9013
## Specificity              0.999714       0.8427      0.7112
## Pos Pred Value           0.954545       0.5129      0.4284
## Neg Pred Value           0.862740       0.9888      0.9677
## Prevalence               0.141667       0.1490      0.1936
## Detection Rate           0.005147       0.1409      0.1745
## Detection Prevalence     0.005392       0.2748      0.4074
## Balanced Accuracy        0.518023       0.8942      0.8063
db_tda_pc_5.50.5_n1_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      150     29   12        0     0     5    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0      963    31    10   75
##   HOROZ           0      0    0        1    21     0    0
##   SEKER         224    127  165       21     6   575    3
##   SIRA           22      0  312       78   520    18  712
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5934          
##                  95% CI : (0.5781, 0.6085)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.4954          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.37879       0.00000      0.0000          0.9059
## Specificity                  0.98751       1.00000      1.0000          0.9616
## Pos Pred Value               0.76531           NaN         NaN          0.8925
## Neg Pred Value               0.93666       0.96176      0.8801          0.9667
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.03676       0.00000      0.0000          0.2360
## Detection Prevalence         0.04804       0.00000      0.0000          0.2645
## Balanced Accuracy            0.68315       0.50000      0.5000          0.9337
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity              0.036332       0.9457      0.9013
## Specificity              0.999714       0.8427      0.7112
## Pos Pred Value           0.954545       0.5129      0.4284
## Neg Pred Value           0.862740       0.9888      0.9677
## Prevalence               0.141667       0.1490      0.1936
## Detection Rate           0.005147       0.1409      0.1745
## Detection Prevalence     0.005392       0.2748      0.4074
## Balanced Accuracy        0.518023       0.8942      0.8063
db_tda_pc_5.50.5_n1_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.5933824      0.4954193      0.5781251      0.6085058      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n1_db_lr_cf0_ov_acc<-db_tda_pc_5.50.5_n1_db_lr_cf0$overall[1]
db_tda_pc_5.50.5_n1_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA  0.37878788   0.9875136      0.7653061      0.9366632 0.7653061
## Class: BOMBAY    0.00000000   1.0000000            NaN      0.9617647        NA
## Class: CALI      0.00000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON  0.90592662   0.9615512      0.8924930      0.9666778 0.8924930
## Class: HOROZ     0.03633218   0.9997144      0.9545455      0.8627403 0.9545455
## Class: SEKER     0.94572368   0.8427419      0.5129349      0.9888476 0.5129349
## Class: SIRA      0.90126582   0.7112462      0.4283995      0.9677419 0.4283995
##                     Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.37878788 0.5067568 0.09705882    0.036764706
## Class: BOMBAY   0.00000000        NA 0.03823529    0.000000000
## Class: CALI     0.00000000        NA 0.11985294    0.000000000
## Class: DERMASON 0.90592662 0.8991597 0.26053922    0.236029412
## Class: HOROZ    0.03633218 0.0700000 0.14166667    0.005147059
## Class: SEKER    0.94572368 0.6651243 0.14901961    0.140931373
## Class: SIRA     0.90126582 0.5807504 0.19362745    0.174509804
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA          0.048039216         0.6831507
## Class: BOMBAY            0.000000000         0.5000000
## Class: CALI              0.000000000         0.5000000
## Class: DERMASON          0.264460784         0.9337389
## Class: HOROZ             0.005392157         0.5180233
## Class: SEKER             0.274754902         0.8942328
## Class: SIRA              0.407352941         0.8062560
db_tda_pc_5.50.5_n1_db_lr_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n1_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted LR vs. tda-assisted LR classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_lr_n1_3_fold<-(db_lr_fit_re - db_tda_pc_5.50.5_n1_lr_fit_re)
diff_drybean_tda_pca_5.50.5_lr_n1_3_fold
##      Accuracy
## 1 0.030925348
## 2 0.010325123
## 3 0.009170028
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n1_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n1_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n1_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n1_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n1_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n1_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n1_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n1_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.3933667
## 
## $winRight
## [1] 0.6066333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n1_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n1_3_fold
## $left
## [1] 0.04075021
## 
## $rope
## [1] 0.2052336
## 
## $right
## [1] 0.7540162
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_lr_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr_n1_3_fold))
#bf_tda_pca_5.50.5_lr.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_lr_n1_3_fold)
## t = 2.3782, df = 2, p-value = 0.1405
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.01360057  0.04721423
## sample estimates:
##  mean of x 
## 0.01680683
### Test set diff
diff_drybean_tda_pca_5.50.5_lr.n1_test<-(db_lr_cf_ov_acc - db_tda_pc_5.50.5_n1_db_lr_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_lr.n1_test
##  Accuracy 
## 0.3267157
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n1_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n1_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n1_test$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n1_test$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n1_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n1_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1614333
## 
## $winRight
## [1] 0.8385667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n1_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_lr.n1_test)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr.n1_test)) #bf_tda_pca_5.50.5_lr.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n1_test))

##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node2

DryBean_TDA_PC_5.50.5_n2_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_dry_bean_dataset_5.50.5.n2.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  126 (102 variable)
## initial  value 12340.962165 
## iter  10 value 7074.107699
## iter  20 value 5048.312161
## iter  30 value 4504.574761
## iter  40 value 2156.660814
## iter  50 value 1801.015774
## iter  60 value 1754.533217
## iter  70 value 1713.391756
## iter  80 value 1701.170874
## iter  90 value 1693.732301
## iter 100 value 1690.816821
## final  value 1690.816821 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12340.962165 
## iter  10 value 7074.107947
## iter  20 value 5048.314715
## iter  30 value 4504.631013
## iter  40 value 2337.731267
## iter  50 value 2000.012357
## iter  60 value 1899.856826
## iter  70 value 1861.319722
## iter  80 value 1854.397204
## iter  90 value 1853.657119
## iter 100 value 1853.137230
## final  value 1853.137230 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12340.962165 
## iter  10 value 7074.107699
## iter  20 value 5048.312170
## iter  30 value 4504.575836
## iter  40 value 2156.118730
## iter  50 value 1802.238357
## iter  60 value 1757.702008
## iter  70 value 1721.197308
## iter  80 value 1711.207258
## iter  90 value 1705.165078
## iter 100 value 1702.772523
## final  value 1702.772523 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12344.853986 
## iter  10 value 7001.946358
## iter  20 value 4750.975536
## iter  30 value 4281.897814
## iter  40 value 2163.324259
## iter  50 value 1714.916709
## iter  60 value 1669.282838
## iter  70 value 1628.082414
## iter  80 value 1608.317145
## iter  90 value 1602.889075
## iter 100 value 1598.384910
## final  value 1598.384910 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12344.853986 
## iter  10 value 7001.946604
## iter  20 value 4750.978616
## iter  30 value 4281.965880
## iter  40 value 2334.883371
## iter  50 value 1922.990025
## iter  60 value 1796.780394
## iter  70 value 1770.984678
## iter  80 value 1757.537897
## iter  90 value 1756.410932
## iter 100 value 1756.080671
## final  value 1756.080671 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 12344.853986 
## iter  10 value 7001.946358
## iter  20 value 4750.975541
## iter  30 value 4281.898263
## iter  40 value 2163.550168
## iter  50 value 1716.120744
## iter  60 value 1672.127829
## iter  70 value 1634.777585
## iter  80 value 1618.411878
## iter  90 value 1614.270289
## iter 100 value 1610.811768
## final  value 1610.811768 
## stopped after 100 iterations
## Warning in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay, :
## group 'BOMBAY' is empty
## # weights:  108 (85 variable)
## initial  value 11366.922073 
## iter  10 value 6113.426403
## iter  20 value 4232.554444
## iter  30 value 2990.316145
## iter  40 value 1871.381646
## iter  50 value 1779.900414
## iter  60 value 1738.983473
## iter  70 value 1721.231790
## iter  80 value 1708.645138
## iter  90 value 1703.861873
## iter 100 value 1700.556767
## final  value 1700.556767 
## stopped after 100 iterations
## Warning in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay, :
## group 'BOMBAY' is empty
## # weights:  108 (85 variable)
## initial  value 11366.922073 
## iter  10 value 6113.426819
## iter  20 value 4232.565129
## iter  30 value 3043.738911
## iter  40 value 2064.143828
## iter  50 value 1930.220649
## iter  60 value 1874.194879
## iter  70 value 1856.809189
## iter  80 value 1853.912580
## iter  90 value 1853.482291
## iter 100 value 1852.860992
## final  value 1852.860992 
## stopped after 100 iterations
## Warning in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay, :
## group 'BOMBAY' is empty
## # weights:  108 (85 variable)
## initial  value 11366.922073 
## iter  10 value 6113.426404
## iter  20 value 4232.554444
## iter  30 value 2990.361566
## iter  40 value 1872.102435
## iter  50 value 1782.303111
## iter  60 value 1743.910679
## iter  70 value 1727.936735
## iter  80 value 1717.297448
## iter  90 value 1713.504586
## iter 100 value 1710.909472
## final  value 1710.909472 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 18515.335068 
## iter  10 value 11489.490912
## iter  20 value 9045.153506
## iter  30 value 7988.113055
## iter  40 value 4033.414254
## iter  50 value 2687.884270
## iter  60 value 2603.938106
## iter  70 value 2545.459568
## iter  80 value 2528.875693
## iter  90 value 2520.071258
## iter 100 value 2514.556975
## final  value 2514.556975 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n2_LrFit0
## Penalized Multinomial Regression 
## 
## 9515 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6342, 6344, 6344 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.9004721  0.8743309
##   1e-04  0.9003668  0.8741878
##   1e-01  0.8962682  0.8690413
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 0.
DryBean_TDA_PC_5.50.5_n2_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.8990855 0.8725637    Fold3
## 2 0.8978240 0.8710404    Fold2
## 3 0.9045068 0.8793886    Fold1
db_tda_pc_5.50.5_n2_lr_fit_re<-DryBean_TDA_PC_5.50.5_n2_LrFit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n2_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##          (Intercept)        Area   Perimeter MajorAxisLength MinorAxisLength
## BOMBAY     16.074628 0.001721945 -0.09625761       0.3037480       0.6718628
## CALI       30.918418 0.003068693 -0.20276057       2.2234406       2.7252793
## DERMASON   -7.635020 0.004152185  0.24362292       0.8490522       1.5633786
## HOROZ       9.005719 0.007163367  0.06060321       2.5190182       4.0217133
## SEKER     -29.580169 0.004796938  0.20455498       0.2908672      -1.0448602
## SIRA       65.101267 0.002586321 -0.44240350       2.3919875       2.6001153
##          AspectRation Eccentricity    ConvexArea EquivDiameter     Extent
## BOMBAY      28.885094     19.18656  0.0006097655    -1.6291048   1.645970
## CALI       -81.578458    102.80003 -0.0037830854    -3.9311565   2.183730
## DERMASON    -9.005589     42.84656 -0.0042641550    -3.3872747 -15.042047
## HOROZ      -27.582461     85.88235 -0.0066387273    -6.9230333  -5.605022
## SEKER      -64.516377    -47.51655 -0.0066522959     0.4765184 -11.675516
## SIRA       -88.449657    110.36087 -0.0047891150    -2.9035403  -6.990231
##           Solidity  roundness Compactness ShapeFactor1 ShapeFactor2
## BOMBAY    15.82905   16.82593   10.317682    0.2955878   0.04388263
## CALI      30.52384  -53.04375    3.815354    0.8162554   0.07676899
## DERMASON -13.97817  152.66681  -17.920757   -0.4401114  -0.39111534
## HOROZ     40.50633   55.29302  -20.973157    0.1465449  -0.27879455
## SEKER    -42.38644  117.39610   16.912407   -0.8957035   0.33345150
## SIRA      38.61104 -175.32440   39.984941    1.2832901   0.34394405
##          ShapeFactor3 ShapeFactor4
## BOMBAY       5.567188    15.143927
## CALI       -28.636315    -6.393010
## DERMASON   -32.207763   -15.426964
## HOROZ      -53.305330    -4.177148
## SEKER       56.829461   -12.958122
## SIRA        -1.007753    19.674859
## 
## Std. Errors:
##           (Intercept)         Area    Perimeter MajorAxisLength MinorAxisLength
## BOMBAY   8.166560e-08 2.653592e-05 3.413936e-05    1.349225e-05    7.683018e-06
## CALI     2.702519e-06 3.717195e-04 1.255802e-03    6.229623e-04    4.586995e-04
## DERMASON 7.238461e-06 6.915100e-04 2.407938e-03    1.474467e-03    1.742129e-03
## HOROZ    3.922213e-06 5.571448e-04 1.697389e-03    6.335806e-04    5.280845e-04
## SEKER    5.258025e-06 7.280802e-04 2.237899e-03    6.496242e-04    6.509797e-04
## SIRA     5.617260e-06 5.303947e-04 1.983924e-03    2.118285e-03    2.116032e-03
##          AspectRation Eccentricity   ConvexArea EquivDiameter       Extent
## BOMBAY   1.392699e-07 6.559283e-08 2.680686e-05  1.016571e-05 6.150724e-08
## CALI     5.278181e-06 2.115903e-06 3.704401e-04  3.851650e-04 2.111512e-06
## DERMASON 1.643336e-05 6.028305e-06 7.048440e-04  8.054496e-04 7.118182e-06
## HOROZ    5.822946e-06 2.897865e-06 5.582560e-04  5.561792e-04 2.722683e-06
## SEKER    6.158551e-06 3.250045e-06 7.342073e-04  6.617309e-04 4.192633e-06
## SIRA     2.241037e-05 7.429005e-06 5.366989e-04  6.909851e-04 6.302452e-06
##              Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## BOMBAY   8.057899e-08 7.115880e-08 6.267958e-08 8.370317e-10 1.825037e-10
## CALI     2.673500e-06 2.596532e-06 2.591972e-06 2.203705e-08 8.408772e-09
## DERMASON 7.154692e-06 8.078973e-06 1.040691e-05 5.935106e-08 5.428715e-08
## HOROZ    3.872327e-06 3.723584e-06 3.368814e-06 3.256429e-08 9.730858e-09
## SEKER    5.202535e-06 4.703929e-06 4.776175e-06 4.517527e-08 1.687011e-08
## SIRA     5.523577e-06 8.127544e-06 1.109684e-05 3.653772e-08 5.982842e-08
##          ShapeFactor3 ShapeFactor4
## BOMBAY   4.821937e-08 8.146744e-08
## CALI     2.576877e-06 2.713459e-06
## DERMASON 1.266472e-05 7.259569e-06
## HOROZ    2.910692e-06 3.907919e-06
## SEKER    4.296500e-06 5.252031e-06
## SIRA     1.458906e-05 5.708986e-06
## 
## Residual Deviance: 5029.114 
## AIC: 5233.114
vip(DryBean_TDA_PC_5.50.5_n2_LrFit0,50) + ggtitle("dryBean_TDA_PCA_5.50.5_n2_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_PC_5.50.5_n2_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n2_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n2_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
db_tda_pc_5.50.5_n2_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   16        1     1     7    3
##   BOMBAY          1    156    0        0     0     0    0
##   CALI           19      0  454        0     9     0    3
##   DERMASON        0      0    0      963     5    12   69
##   HOROZ           3      0   11        3   553     0    8
##   SEKER           2      0    1       16     0   572    4
##   SIRA            7      0    7       80    10    17  703
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9228          
##                  95% CI : (0.9142, 0.9308)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.9067          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000      0.9284          0.9059
## Specificity                  0.99240       0.99975      0.9914          0.9715
## Pos Pred Value               0.92857       0.99363      0.9361          0.9180
## Neg Pred Value               0.99132       1.00000      0.9903          0.9670
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.03824      0.1113          0.2360
## Detection Prevalence         0.09608       0.03848      0.1189          0.2571
## Balanced Accuracy            0.95580       0.99987      0.9599          0.9387
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9567       0.9408      0.8899
## Specificity                0.9929       0.9934      0.9632
## Pos Pred Value             0.9567       0.9613      0.8532
## Neg Pred Value             0.9929       0.9897      0.9733
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1355       0.1402      0.1723
## Detection Prevalence       0.1417       0.1458      0.2020
## Balanced Accuracy          0.9748       0.9671      0.9265
db_tda_pc_5.50.5_n2_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   16        1     1     7    3
##   BOMBAY          1    156    0        0     0     0    0
##   CALI           19      0  454        0     9     0    3
##   DERMASON        0      0    0      963     5    12   69
##   HOROZ           3      0   11        3   553     0    8
##   SEKER           2      0    1       16     0   572    4
##   SIRA            7      0    7       80    10    17  703
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9228          
##                  95% CI : (0.9142, 0.9308)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.9067          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000      0.9284          0.9059
## Specificity                  0.99240       0.99975      0.9914          0.9715
## Pos Pred Value               0.92857       0.99363      0.9361          0.9180
## Neg Pred Value               0.99132       1.00000      0.9903          0.9670
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.03824      0.1113          0.2360
## Detection Prevalence         0.09608       0.03848      0.1189          0.2571
## Balanced Accuracy            0.95580       0.99987      0.9599          0.9387
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9567       0.9408      0.8899
## Specificity                0.9929       0.9934      0.9632
## Pos Pred Value             0.9567       0.9613      0.8532
## Neg Pred Value             0.9929       0.9897      0.9733
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1355       0.1402      0.1723
## Detection Prevalence       0.1417       0.1458      0.2020
## Balanced Accuracy          0.9748       0.9671      0.9265
db_tda_pc_5.50.5_n2_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.9227941      0.9066505      0.9141749      0.9308019      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n2_db_lr_cf0_ov_acc<-db_tda_pc_5.50.5_n2_db_lr_cf0$overall[1]
db_tda_pc_5.50.5_n2_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9191919   0.9923996      0.9285714      0.9913232 0.9285714
## Class: BOMBAY     1.0000000   0.9997452      0.9936306      1.0000000 0.9936306
## Class: CALI       0.9284254   0.9913673      0.9360825      0.9902643 0.9360825
## Class: DERMASON   0.9059266   0.9714949      0.9180172      0.9670076 0.9180172
## Class: HOROZ      0.9567474   0.9928612      0.9567474      0.9928612 0.9567474
## Class: SEKER      0.9407895   0.9933756      0.9613445      0.9896700 0.9613445
## Class: SIRA       0.8898734   0.9632219      0.8531553      0.9732801 0.8531553
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9191919 0.9238579 0.09705882     0.08921569
## Class: BOMBAY   1.0000000 0.9968051 0.03823529     0.03823529
## Class: CALI     0.9284254 0.9322382 0.11985294     0.11127451
## Class: DERMASON 0.9059266 0.9119318 0.26053922     0.23602941
## Class: HOROZ    0.9567474 0.9567474 0.14166667     0.13553922
## Class: SEKER    0.9407895 0.9509559 0.14901961     0.14019608
## Class: SIRA     0.8898734 0.8711276 0.19362745     0.17230392
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09607843         0.9557957
## Class: BOMBAY             0.03848039         0.9998726
## Class: CALI               0.11887255         0.9598963
## Class: DERMASON           0.25710784         0.9387107
## Class: HOROZ              0.14166667         0.9748043
## Class: SEKER              0.14583333         0.9670825
## Class: SIRA               0.20196078         0.9265477
db_tda_pc_5.50.5_n2_db_lr_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n2_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted LR vs. tda-assisted LR classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_lr_n2_3_fold<-(db_lr_fit_re - db_tda_pc_5.50.5_n2_lr_fit_re)
diff_drybean_tda_pca_5.50.5_lr_n2_3_fold
##     Accuracy
## 1 0.03731252
## 2 0.02256615
## 3 0.01932073
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n2_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n2_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n2_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n2_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n2_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n2_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n2_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n2_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n2_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n2_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.03696667
## 
## $winRight
## [1] 0.9630333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n2_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n2_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n2_3_fold
## $left
## [1] 0.01474312
## 
## $rope
## [1] 0.04738248
## 
## $right
## [1] 0.9378744
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_lr_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr_n2_3_fold))
#bf_tda_pca_5.50.5_lr.n2_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n2_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_lr_n2_3_fold)
## t = 4.7686, df = 2, p-value = 0.04127
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.002579426 0.050220181
## sample estimates:
## mean of x 
## 0.0263998
### Test set diff
diff_drybean_tda_pca_5.50.5_lr.n2_test<-(db_lr_cf_ov_acc - db_tda_pc_5.50.5_n2_db_lr_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_lr.n2_test
##     Accuracy 
## -0.002696078
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n2_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n2_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 1
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n2_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n2_test$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n2_test$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n2_test_odds.left
## [1] NaN
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n2_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 1
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n2_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n2_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n2_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_lr.n2_test)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr.n2_test)) #bf_tda_pca_5.50.5_lr.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n2_test))

##Node3

DryBean_TDA_PC_5.50.5_n3_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_dry_bean_dataset_5.50.5.n3.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 4514.997621
## iter  20 value 2951.751507
## iter  30 value 2436.112764
## iter  40 value 801.001707
## iter  50 value 556.327157
## iter  60 value 541.977532
## iter  70 value 534.175569
## iter  80 value 527.790947
## iter  90 value 524.316187
## iter 100 value 522.121591
## final  value 522.121591 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 4514.997698
## iter  20 value 2951.765476
## iter  30 value 2436.367014
## iter  40 value 873.878393
## iter  50 value 647.130147
## iter  60 value 605.776183
## iter  70 value 590.751820
## iter  80 value 588.105743
## iter  90 value 586.443548
## iter 100 value 585.006086
## final  value 585.006086 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 4514.997624
## iter  20 value 2951.752914
## iter  30 value 2436.147309
## iter  40 value 800.901379
## iter  50 value 556.791295
## iter  60 value 542.832332
## iter  70 value 535.818817
## iter  80 value 531.017896
## iter  90 value 528.668068
## iter 100 value 527.366451
## final  value 527.366451 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 3869.823349
## iter  20 value 2201.134171
## iter  30 value 1887.819960
## iter  40 value 812.503341
## iter  50 value 587.866312
## iter  60 value 569.347098
## iter  70 value 558.741299
## iter  80 value 550.564403
## iter  90 value 542.504291
## iter 100 value 539.417432
## final  value 539.417432 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 3869.823410
## iter  20 value 2201.145829
## iter  30 value 1887.955038
## iter  40 value 964.306843
## iter  50 value 653.220706
## iter  60 value 619.668800
## iter  70 value 611.611444
## iter  80 value 609.407498
## iter  90 value 608.222845
## iter 100 value 607.270427
## final  value 607.270427 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 3869.823349
## iter  20 value 2201.134175
## iter  30 value 1887.820453
## iter  40 value 812.681840
## iter  50 value 588.460774
## iter  60 value 570.490550
## iter  70 value 560.891150
## iter  80 value 554.014623
## iter  90 value 547.908858
## iter 100 value 545.984264
## final  value 545.984264 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 4236.257154
## iter  20 value 2607.394196
## iter  30 value 2130.988976
## iter  40 value 816.099481
## iter  50 value 564.068075
## iter  60 value 551.110729
## iter  70 value 542.163309
## iter  80 value 537.312876
## iter  90 value 530.721770
## iter 100 value 525.261849
## final  value 525.261849 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 4236.257165
## iter  20 value 2607.399227
## iter  30 value 2131.144300
## iter  40 value 897.212419
## iter  50 value 654.262126
## iter  60 value 613.218418
## iter  70 value 592.419456
## iter  80 value 589.809006
## iter  90 value 589.026648
## iter 100 value 587.602478
## final  value 587.602478 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 6946.899232 
## iter  10 value 4236.257154
## iter  20 value 2607.394249
## iter  30 value 2130.996984
## iter  40 value 816.975911
## iter  50 value 564.904001
## iter  60 value 552.174775
## iter  70 value 544.096230
## iter  80 value 540.082746
## iter  90 value 535.062639
## iter 100 value 532.237558
## final  value 532.237558 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10420.348848 
## iter  10 value 6845.310431
## iter  20 value 3500.938717
## iter  30 value 2783.052684
## iter  40 value 1069.786186
## iter  50 value 874.975492
## iter  60 value 844.382610
## iter  70 value 828.321111
## iter  80 value 815.153614
## iter  90 value 807.463694
## iter 100 value 802.936198
## final  value 802.936198 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n3_LrFit0
## Penalized Multinomial Regression 
## 
## 5355 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 3570, 3570, 3570 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.9449113  0.9232484
##   1e-04  0.9447246  0.9229828
##   1e-01  0.9415500  0.9184811
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 0.
DryBean_TDA_PC_5.50.5_n3_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9450980 0.9237105    Fold3
## 2 0.9439776 0.9218613    Fold2
## 3 0.9456583 0.9241733    Fold1
db_tda_pc_5.50.5_n3_lr_fit_re<-DryBean_TDA_PC_5.50.5_n2_LrFit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n3_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##          (Intercept)        Area   Perimeter MajorAxisLength MinorAxisLength
## BOMBAY       9.85067 0.009119522 -0.09002700        1.169633        2.586884
## CALI        21.37667 0.002922220 -0.17027119        1.753240        2.632045
## DERMASON    22.21659 0.003894321 -0.01368438        1.089894        1.943034
## HOROZ      -13.79197 0.006834102  0.10253300        1.745168        4.070059
## SEKER       34.04545 0.002625139 -0.05243008        1.237072        1.783588
## SIRA        14.84737 0.001692132  0.00573875        1.986698        1.726498
##          AspectRation Eccentricity   ConvexArea EquivDiameter     Extent
## BOMBAY       56.69264     18.78195 -0.007113250     -4.135310  -2.918904
## CALI        -58.76140    130.59714 -0.003342126     -3.494826   3.857366
## DERMASON     35.33526     14.83545 -0.001742719     -3.979289   5.784793
## HOROZ        30.97531     88.83514 -0.006420320     -6.030582  -5.149266
## SEKER        -6.48801     33.72721 -0.001193855     -3.537147 -35.851309
## SIRA        -97.22407     58.72235 -0.004063682     -3.187320  -9.216045
##           Solidity  roundness  Compactness ShapeFactor1 ShapeFactor2
## BOMBAY    6.934488  -1.169424  -0.01176916    0.2343526  -0.03927711
## CALI     31.091798 -40.278082 -17.89230207    0.4244595  -0.27333353
## DERMASON 17.589892   1.429623  19.06426277    0.4377699   0.12210271
## HOROZ     8.536009  79.062409 -57.10920381    0.3085262  -0.51018483
## SEKER    19.186947  12.556114  29.66440801    0.4275961   0.17717002
## SIRA     12.789659  71.467933  -0.59003747   -0.1299115  -0.13237123
##          ShapeFactor3 ShapeFactor4
## BOMBAY      -5.373915     9.751554
## CALI       -60.381695   -15.088440
## DERMASON    16.658093    23.049217
## HOROZ      -97.473939   -28.792974
## SEKER       20.573700    21.470160
## SIRA       -23.906360   -14.138014
## 
## Std. Errors:
##           (Intercept)         Area    Perimeter MajorAxisLength MinorAxisLength
## BOMBAY   8.670993e-07 0.0001788435 0.0004640906    1.849404e-04    9.020876e-05
## CALI     9.401235e-06 0.0002998468 0.0010982449    3.424343e-03    4.093801e-03
## DERMASON 1.095476e-07 0.0021392502 0.0002046423    4.544822e-05    5.100414e-05
## HOROZ    4.527463e-06 0.0003823974 0.0019055272    2.083366e-03    1.838130e-03
## SEKER    5.022275e-07 0.0009646133 0.0002550029    9.421915e-05    4.384243e-05
## SIRA     6.105328e-06 0.0005528918 0.0028354980    9.592361e-04    6.566459e-04
##          AspectRation Eccentricity   ConvexArea EquivDiameter       Extent
## BOMBAY   1.731974e-06 7.340871e-07 0.0001701765  1.282997e-04 6.511723e-07
## CALI     3.085479e-05 4.581312e-06 0.0002978887  1.385375e-03 8.791476e-06
## DERMASON 5.720006e-07 1.467320e-07 0.0020332314  6.326258e-06 7.036579e-08
## HOROZ    1.949012e-05 4.010235e-06 0.0003794985  6.622067e-04 3.308819e-06
## SEKER    9.591364e-07 4.388559e-07 0.0009436872  6.149589e-05 3.219585e-07
## SIRA     1.004507e-05 4.829997e-06 0.0005591507  7.738328e-04 4.397460e-06
##              Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## BOMBAY   8.532193e-07 6.876297e-07 6.187827e-07 8.004086e-09 1.316262e-09
## CALI     9.263770e-06 1.529659e-05 1.730632e-05 1.772266e-08 6.477991e-08
## DERMASON 1.516780e-08 2.379575e-07 2.349288e-07 1.147431e-10 1.317496e-09
## HOROZ    4.462579e-06 6.233295e-06 7.719062e-06 3.167198e-08 2.918906e-08
## SEKER    4.757019e-07 3.606177e-07 3.616405e-07 5.296474e-09 1.013266e-09
## SIRA     6.016073e-06 4.838463e-06 4.891159e-06 5.933683e-08 1.388876e-08
##          ShapeFactor3 ShapeFactor4
## BOMBAY   4.454696e-07 8.648184e-07
## CALI     2.100637e-05 9.204588e-06
## DERMASON 2.990672e-07 3.627933e-08
## HOROZ    9.419990e-06 4.435812e-06
## SEKER    2.630352e-07 4.871964e-07
## SIRA     3.969653e-06 6.147876e-06
## 
## Residual Deviance: 1605.872 
## AIC: 1809.872
vip(DryBean_TDA_PC_5.50.5_n3_LrFit0,50) + ggtitle("dryBean_TDA_PCA_5.50.5_n3_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_PC_5.50.5_n3_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n3_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n3_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
db_tda_pc_5.50.5_n3_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      366      0   17        1     1    22    5
##   BOMBAY          0    154    0        0     0     0    0
##   CALI           17      0  458        0    10     0    3
##   DERMASON        0      2    0       66     0     0    0
##   HOROZ           4      0   10        2   553     0   11
##   SEKER           1      0    0      189     0   359    0
##   SIRA            8      0    4      805    14   227  771
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6684          
##                  95% CI : (0.6537, 0.6828)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.6067          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.92424       0.98718      0.9366         0.06209
## Specificity                  0.98751       1.00000      0.9916         0.99934
## Pos Pred Value               0.88835       1.00000      0.9385         0.97059
## Neg Pred Value               0.99182       0.99949      0.9914         0.75150
## Prevalence                   0.09706       0.03824      0.1199         0.26054
## Detection Rate               0.08971       0.03775      0.1123         0.01618
## Detection Prevalence         0.10098       0.03775      0.1196         0.01667
## Balanced Accuracy            0.95588       0.99359      0.9641         0.53071
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9567      0.59046      0.9759
## Specificity                0.9923      0.94528      0.6784
## Pos Pred Value             0.9534      0.65392      0.4215
## Neg Pred Value             0.9929      0.92948      0.9916
## Prevalence                 0.1417      0.14902      0.1936
## Detection Rate             0.1355      0.08799      0.1890
## Detection Prevalence       0.1422      0.13456      0.4483
## Balanced Accuracy          0.9745      0.76787      0.8272
db_tda_pc_5.50.5_n3_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      366      0   17        1     1    22    5
##   BOMBAY          0    154    0        0     0     0    0
##   CALI           17      0  458        0    10     0    3
##   DERMASON        0      2    0       66     0     0    0
##   HOROZ           4      0   10        2   553     0   11
##   SEKER           1      0    0      189     0   359    0
##   SIRA            8      0    4      805    14   227  771
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6684          
##                  95% CI : (0.6537, 0.6828)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.6067          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.92424       0.98718      0.9366         0.06209
## Specificity                  0.98751       1.00000      0.9916         0.99934
## Pos Pred Value               0.88835       1.00000      0.9385         0.97059
## Neg Pred Value               0.99182       0.99949      0.9914         0.75150
## Prevalence                   0.09706       0.03824      0.1199         0.26054
## Detection Rate               0.08971       0.03775      0.1123         0.01618
## Detection Prevalence         0.10098       0.03775      0.1196         0.01667
## Balanced Accuracy            0.95588       0.99359      0.9641         0.53071
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9567      0.59046      0.9759
## Specificity                0.9923      0.94528      0.6784
## Pos Pred Value             0.9534      0.65392      0.4215
## Neg Pred Value             0.9929      0.92948      0.9916
## Prevalence                 0.1417      0.14902      0.1936
## Detection Rate             0.1355      0.08799      0.1890
## Detection Prevalence       0.1422      0.13456      0.4483
## Balanced Accuracy          0.9745      0.76787      0.8272
db_tda_pc_5.50.5_n3_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.6683824      0.6066627      0.6536971      0.6828261      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n3_db_lr_cf0_ov_acc<-db_tda_pc_5.50.5_n3_db_lr_cf0$overall[1]
db_tda_pc_5.50.5_n3_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA  0.92424242   0.9875136      0.8883495      0.9918212 0.8883495
## Class: BOMBAY    0.98717949   1.0000000      1.0000000      0.9994906 1.0000000
## Class: CALI      0.93660532   0.9916458      0.9385246      0.9913697 0.9385246
## Class: DERMASON  0.06208843   0.9993371      0.9705882      0.7514955 0.9705882
## Class: HOROZ     0.95674740   0.9922901      0.9534483      0.9928571 0.9534483
## Class: SEKER     0.59046053   0.9452765      0.6539162      0.9294817 0.6539162
## Class: SIRA      0.97594937   0.6784195      0.4215418      0.9915593 0.4215418
##                     Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.92424242 0.9059406 0.09705882     0.08970588
## Class: BOMBAY   0.98717949 0.9935484 0.03823529     0.03774510
## Class: CALI     0.93660532 0.9375640 0.11985294     0.11225490
## Class: DERMASON 0.06208843 0.1167109 0.26053922     0.01617647
## Class: HOROZ    0.95674740 0.9550950 0.14166667     0.13553922
## Class: SEKER    0.59046053 0.6205704 0.14901961     0.08799020
## Class: SIRA     0.97594937 0.5887743 0.19362745     0.18897059
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.10098039         0.9558780
## Class: BOMBAY             0.03774510         0.9935897
## Class: CALI               0.11960784         0.9641255
## Class: DERMASON           0.01666667         0.5307128
## Class: HOROZ              0.14215686         0.9745188
## Class: SEKER              0.13455882         0.7678685
## Class: SIRA               0.44828431         0.8271844
db_tda_pc_5.50.5_n3_db_lr_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n3_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted LR vs. tda-assisted LR classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_lr_n3_3_fold<-(db_lr_fit_re - db_tda_pc_5.50.5_n3_lr_fit_re)
diff_drybean_tda_pca_5.50.5_lr_n3_3_fold
##     Accuracy
## 1 0.03731252
## 2 0.02256615
## 3 0.01932073
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n3_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n3_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n3_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n3_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n3_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n3_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n3_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n3_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n3_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n3_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.0377
## 
## $winRight
## [1] 0.9623
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n3_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n3_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n3_3_fold
## $left
## [1] 0.01474312
## 
## $rope
## [1] 0.04738248
## 
## $right
## [1] 0.9378744
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_lr_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr_n3_3_fold))
#bf_tda_pca_5.50.5_lr.n3_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n3_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_lr_n3_3_fold)
## t = 4.7686, df = 2, p-value = 0.04127
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.002579426 0.050220181
## sample estimates:
## mean of x 
## 0.0263998
### Test set diff
diff_drybean_tda_pca_5.50.5_lr.n3_test<-(db_lr_cf_ov_acc - db_tda_pc_5.50.5_n3_db_lr_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_lr.n3_test
##  Accuracy 
## 0.2517157
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n3_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n3_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n3_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n3_test$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n3_test$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n3_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n3_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n3_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n3_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1568667
## 
## $winRight
## [1] 0.8431333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n3_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n3_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n3_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_lr.n3_test)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr.n3_test)) #bf_tda_pca_5.50.5_lr.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n3_test))

##Node4

DryBean_TDA_PC_5.50.5_n4_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_dry_bean_dataset_5.50.5.n4.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  72 (51 variable)
## initial  value 1470.858317 
## iter  10 value 618.079432
## iter  20 value 212.550525
## iter  30 value 50.363671
## iter  40 value 48.230025
## iter  50 value 47.343623
## iter  60 value 46.789835
## iter  70 value 46.542332
## iter  80 value 46.361652
## iter  90 value 46.259890
## iter 100 value 45.920749
## final  value 45.920749 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1470.858317 
## iter  10 value 618.079493
## iter  20 value 215.975148
## iter  30 value 80.350523
## iter  40 value 62.019205
## iter  50 value 57.853443
## iter  60 value 57.514581
## iter  70 value 57.498470
## iter  80 value 57.491022
## iter  90 value 57.476253
## iter 100 value 57.423924
## final  value 57.423924 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1470.858317 
## iter  10 value 618.079432
## iter  20 value 212.552510
## iter  30 value 50.599813
## iter  40 value 48.477229
## iter  50 value 47.578097
## iter  60 value 47.064273
## iter  70 value 46.886634
## iter  80 value 46.801554
## iter  90 value 46.740556
## iter 100 value 46.501255
## final  value 46.501255 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1468.085728 
## iter  10 value 650.446414
## iter  20 value 140.642786
## iter  30 value 55.600797
## iter  40 value 52.581913
## iter  50 value 51.503603
## iter  60 value 50.824243
## iter  70 value 50.004078
## iter  80 value 48.626175
## iter  90 value 47.929465
## iter 100 value 47.830153
## final  value 47.830153 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1468.085728 
## iter  10 value 650.446464
## iter  20 value 178.001562
## iter  30 value 83.794914
## iter  40 value 65.150422
## iter  50 value 61.157328
## iter  60 value 59.987595
## iter  70 value 59.713364
## iter  80 value 59.677665
## iter  90 value 59.667228
## iter 100 value 59.661004
## final  value 59.661004 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1468.085728 
## iter  10 value 650.446414
## iter  20 value 140.681139
## iter  30 value 55.790161
## iter  40 value 52.836606
## iter  50 value 51.825845
## iter  60 value 51.234492
## iter  70 value 50.630508
## iter  80 value 49.966557
## iter  90 value 49.744692
## iter 100 value 49.715354
## final  value 49.715354 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1469.472023 
## iter  10 value 658.560214
## iter  20 value 182.787195
## iter  30 value 68.150202
## iter  40 value 65.944292
## iter  50 value 64.388886
## iter  60 value 63.609920
## iter  70 value 62.805373
## iter  80 value 62.175524
## iter  90 value 61.879703
## iter 100 value 61.045030
## final  value 61.045030 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1469.472023 
## iter  10 value 658.560402
## iter  20 value 186.360584
## iter  30 value 93.889368
## iter  40 value 78.692496
## iter  50 value 72.713399
## iter  60 value 72.319048
## iter  70 value 72.243787
## iter  80 value 72.239189
## iter  90 value 72.237709
## iter 100 value 72.230356
## final  value 72.230356 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1469.472023 
## iter  10 value 658.560214
## iter  20 value 182.793716
## iter  30 value 68.305306
## iter  40 value 66.104387
## iter  50 value 64.620994
## iter  60 value 63.968998
## iter  70 value 63.475439
## iter  80 value 63.134330
## iter  90 value 62.947955
## iter 100 value 62.654666
## final  value 62.654666 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 2204.208034 
## iter  10 value 783.542866
## iter  20 value 258.362081
## iter  30 value 89.910550
## iter  40 value 86.689336
## iter  50 value 84.599279
## iter  60 value 83.543152
## iter  70 value 83.047437
## iter  80 value 82.871730
## iter  90 value 82.350165
## iter 100 value 82.223466
## final  value 82.223466 
## stopped after 100 iterations
DryBean_TDA_PC_5.50.5_n4_LrFit0
## Penalized Multinomial Regression 
## 
## 1590 samples
##   16 predictor
##    4 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'HOROZ' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1061, 1059, 1060 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.9805043  0.9715922
##   1e-04  0.9836478  0.9761486
##   1e-01  0.9805043  0.9716044
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 1e-04.
DryBean_TDA_PC_5.50.5_n4_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9811676 0.9724551    Fold2
## 2 0.9810964 0.9723895    Fold1
## 3 0.9886792 0.9836012    Fold3
db_tda_pc_5.50.5_n4_lr_fit_re<-DryBean_TDA_PC_5.50.5_n4_LrFit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n4_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##        (Intercept)         Area   Perimeter MajorAxisLength MinorAxisLength
## BOMBAY   -4.159853 0.0010347292 -0.16514095       0.8722229        1.756878
## CALI     -0.793122 0.0006979468 -0.09189237       1.0563761        1.124534
## HOROZ    12.082874 0.0069799208  0.03994281       2.1578960        3.246367
##        AspectRation Eccentricity    ConvexArea EquivDiameter    Extent
## BOMBAY     23.82217    -3.695187 -0.0001276862     -2.285295 18.547615
## CALI      -46.94882    40.309977 -0.0011919297     -1.663021  7.800909
## HOROZ      10.09107   -24.254603 -0.0062027749     -5.921399 -8.331375
##         Solidity roundness Compactness ShapeFactor1 ShapeFactor2 ShapeFactor3
## BOMBAY -3.853775 -6.341031   -5.713175   0.08007980  -0.02300731    -4.593631
## CALI   21.718682 17.432279  -14.886856  -0.17804373  -0.14973099   -31.699405
## HOROZ   1.016797 24.252263   26.552978   0.07184372   0.19499819    37.344327
##        ShapeFactor4
## BOMBAY    -2.145152
## CALI      -6.046346
## HOROZ     19.001400
## 
## Std. Errors:
##         (Intercept)         Area    Perimeter MajorAxisLength MinorAxisLength
## BOMBAY 7.018313e-07 0.0052243255 0.0008108686    1.744398e-05    0.0002134523
## CALI   4.674780e-06 0.0006076085 0.0031843724    1.422169e-03    0.0003467775
## HOROZ  6.289482e-06 0.0007130044 0.0052699597    1.474966e-03    0.0004951484
##        AspectRation Eccentricity   ConvexArea EquivDiameter       Extent
## BOMBAY 1.490428e-07 1.228230e-07 0.0051306014  0.0001281071 2.643921e-07
## CALI   1.158052e-05 4.592335e-06 0.0006205885  0.0007541547 3.049354e-06
## HOROZ  1.417523e-05 5.851883e-06 0.0007373128  0.0009089062 5.275184e-06
##            Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## BOMBAY 6.036271e-07 1.313798e-07 8.152171e-07 3.192879e-09 2.151808e-09
## CALI   4.586799e-06 3.038004e-06 2.799510e-06 4.224410e-08 4.363133e-09
## HOROZ  6.123367e-06 2.324847e-06 4.143589e-06 6.082113e-08 7.965245e-09
##        ShapeFactor3 ShapeFactor4
## BOMBAY 8.671634e-07 6.635749e-07
## CALI   1.586586e-06 4.608469e-06
## HOROZ  2.684442e-06 6.420998e-06
## 
## Residual Deviance: 164.4469 
## AIC: 266.4469
vip(DryBean_TDA_PC_5.50.5_n4_LrFit0,50) + ggtitle("dryBean_TDA_PCA_5.50.5_n4_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_PC_5.50.5_n4_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n4_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n4_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n4_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      314      0    5        0     1     1    0
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           32      0  472        0    14     1   28
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ          50      0   12     1063   563   606  762
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                          
##                Accuracy : 0.3689         
##                  95% CI : (0.354, 0.3839)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.2735         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.79293       1.00000      0.9652          0.0000
## Specificity                  0.99810       1.00000      0.9791          1.0000
## Pos Pred Value               0.97819       1.00000      0.8629             NaN
## Neg Pred Value               0.97819       1.00000      0.9952          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.07696       0.03824      0.1157          0.0000
## Detection Prevalence         0.07868       0.03824      0.1341          0.0000
## Balanced Accuracy            0.89551       1.00000      0.9722          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9740        0.000      0.0000
## Specificity                0.2881        1.000      1.0000
## Pos Pred Value             0.1842          NaN         NaN
## Neg Pred Value             0.9854        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1380        0.000      0.0000
## Detection Prevalence       0.7490        0.000      0.0000
## Balanced Accuracy          0.6311        0.500      0.5000
db_tda_pc_5.50.5_n4_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      314      0    5        0     1     1    0
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           32      0  472        0    14     1   28
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ          50      0   12     1063   563   606  762
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                          
##                Accuracy : 0.3689         
##                  95% CI : (0.354, 0.3839)
##     No Information Rate : 0.2605         
##     P-Value [Acc > NIR] : < 2.2e-16      
##                                          
##                   Kappa : 0.2735         
##                                          
##  Mcnemar's Test P-Value : NA             
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.79293       1.00000      0.9652          0.0000
## Specificity                  0.99810       1.00000      0.9791          1.0000
## Pos Pred Value               0.97819       1.00000      0.8629             NaN
## Neg Pred Value               0.97819       1.00000      0.9952          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.07696       0.03824      0.1157          0.0000
## Detection Prevalence         0.07868       0.03824      0.1341          0.0000
## Balanced Accuracy            0.89551       1.00000      0.9722          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9740        0.000      0.0000
## Specificity                0.2881        1.000      1.0000
## Pos Pred Value             0.1842          NaN         NaN
## Neg Pred Value             0.9854        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1380        0.000      0.0000
## Detection Prevalence       0.7490        0.000      0.0000
## Balanced Accuracy          0.6311        0.500      0.5000
db_tda_pc_5.50.5_n4_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   3.688725e-01   2.734991e-01   3.540431e-01   3.838900e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##   2.551451e-52            NaN
db_tda_pc_5.50.5_n4_db_lr_cf0_ov_acc<-db_tda_pc_5.50.5_n4_db_lr_cf0$overall[1]
db_tda_pc_5.50.5_n4_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.7929293   0.9980999      0.9781931      0.9781857 0.9781931
## Class: BOMBAY     1.0000000   1.0000000      1.0000000      1.0000000 1.0000000
## Class: CALI       0.9652352   0.9791145      0.8628885      0.9951882 0.8628885
## Class: DERMASON   0.0000000   1.0000000            NaN      0.7394608        NA
## Class: HOROZ      0.9740484   0.2881211      0.1842277      0.9853516 0.1842277
## Class: SEKER      0.0000000   1.0000000            NaN      0.8509804        NA
## Class: SIRA       0.0000000   1.0000000            NaN      0.8063725        NA
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.7929293 0.8758717 0.09705882     0.07696078
## Class: BOMBAY   1.0000000 1.0000000 0.03823529     0.03823529
## Class: CALI     0.9652352 0.9111969 0.11985294     0.11568627
## Class: DERMASON 0.0000000        NA 0.26053922     0.00000000
## Class: HOROZ    0.9740484 0.3098514 0.14166667     0.13799020
## Class: SEKER    0.0000000        NA 0.14901961     0.00000000
## Class: SIRA     0.0000000        NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.07867647         0.8955146
## Class: BOMBAY             0.03823529         1.0000000
## Class: CALI               0.13406863         0.9721748
## Class: DERMASON           0.00000000         0.5000000
## Class: HOROZ              0.74901961         0.6310848
## Class: SEKER              0.00000000         0.5000000
## Class: SIRA               0.00000000         0.5000000
db_tda_pc_5.50.5_n4_db_lr_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n4_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted LR vs. tda-assisted LR classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_lr_n4_3_fold<-(db_lr_fit_re - db_tda_pc_5.50.5_n4_lr_fit_re)
diff_drybean_tda_pca_5.50.5_lr_n4_3_fold
##      Accuracy
## 1 -0.04476962
## 2 -0.06070623
## 3 -0.06485174
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n4_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n4_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n4_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n4_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n4_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n4_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n4_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n4_3_fold
## $winLeft
## [1] 0.9918667
## 
## $winRope
## [1] 0.008133333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n4_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n4_3_fold
## $left
## [1] 0.9889599
## 
## $rope
## [1] 0.005530419
## 
## $right
## [1] 0.00550964
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_lr_n4_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr_n4_3_fold))
#bf_tda_pca_5.50.5_lr.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_lr_n4_3_fold)
## t = -9.2752, df = 2, p-value = 0.01143
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.08311342 -0.03043830
## sample estimates:
##   mean of x 
## -0.05677586
### Test set diff
diff_drybean_tda_pca_5.50.5_lr.n4_test<-(db_lr_cf_ov_acc - db_tda_pc_5.50.5_n4_db_lr_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_lr.n4_test
##  Accuracy 
## 0.5512255
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n4_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n4_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n4_test$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n4_test$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n4_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1607667
## 
## $winRight
## [1] 0.8392333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n4_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_lr.n4_test)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr.n4_test)) #bf_tda_pca_5.50.5_lr.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n4_test))

##Node5

DryBean_TDA_PC_5.50.5_n5_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_dry_bean_dataset_5.50.5.n5.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  18 (17 variable)
## initial  value 192.694916 
## final  value 0.000000 
## converged
## # weights:  18 (17 variable)
## initial  value 192.694916 
## iter  10 value 0.001391
## iter  20 value 0.001282
## iter  30 value 0.001280
## iter  40 value 0.000407
## iter  50 value 0.000321
## final  value 0.000321 
## converged
## # weights:  18 (17 variable)
## initial  value 192.694916 
## final  value 0.000037 
## converged
## Warning in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay, :
## group 'CALI' is empty
## Warning: model fit failed for Fold2: decay=0e+00 Error in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay,  : 
##   need two or more classes to fit a multinom model
## Warning in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay, :
## group 'CALI' is empty
## Warning: model fit failed for Fold2: decay=1e-01 Error in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay,  : 
##   need two or more classes to fit a multinom model
## Warning in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay, :
## group 'CALI' is empty
## Warning: model fit failed for Fold2: decay=1e-04 Error in nnet::multinom(.outcome ~ ., data = dat, decay = param$decay,  : 
##   need two or more classes to fit a multinom model
## # weights:  18 (17 variable)
## initial  value 192.694916 
## final  value 184.206807 
## converged
## # weights:  18 (17 variable)
## initial  value 192.694916 
## final  value 184.206808 
## converged
## # weights:  18 (17 variable)
## initial  value 192.694916 
## final  value 184.206807 
## converged
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo,
## : There were missing values in resampled performance measures.
## # weights:  18 (17 variable)
## initial  value 289.042374 
## final  value 184.206807 
## converged
DryBean_TDA_PC_5.50.5_n5_LrFit0
## Penalized Multinomial Regression 
## 
## 417 samples
##  16 predictor
##   2 classes: 'BOMBAY', 'CALI' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 278, 278, 278 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa
##   0e+00  1.0000000  NaN  
##   1e-04  1.0000000  NaN  
##   1e-01  0.9928058    0  
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 1e-04.
DryBean_TDA_PC_5.50.5_n5_LrFit0$resample
##   Accuracy Kappa Resample
## 1       NA    NA    Fold2
## 2        1    NA    Fold1
## 3        1    NA    Fold3
db_tda_pc_5.50.5_n5_lr_fit_re<-DryBean_TDA_PC_5.50.5_n5_LrFit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n5_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##                        Values Std. Err.
## (Intercept)     -3.399680e-08         0
## Area            -6.143051e-03         0
## Perimeter       -5.523928e-05         0
## MajorAxisLength -2.079644e-05         0
## MinorAxisLength -1.287831e-05         0
## AspectRation    -5.497611e-08         0
## Eccentricity    -2.659165e-08         0
## ConvexArea      -6.229317e-03         0
## EquivDiameter   -1.628301e-05         0
## Extent          -2.639320e-08         0
## Solidity        -3.352785e-08         0
## roundness       -2.917612e-08         0
## Compactness     -2.665608e-08         0
## ShapeFactor1    -1.156720e-10         0
## ShapeFactor2    -2.706018e-11         0
## ShapeFactor3    -2.092300e-08         0
## ShapeFactor4    -3.369234e-08         0
## 
## Residual Deviance: 368.4136 
## AIC: 402.4136
vip(DryBean_TDA_PC_5.50.5_n5_LrFit0,50) + ggtitle("dryBean_TDA_PCA_5.50.5_n5_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_PC_5.50.5_n5_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n5_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n5_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n5_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY        396    156  489     1063   578   608  790
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.0382          
##                  95% CI : (0.0326, 0.0446)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000      0.0000          0.0000
## Specificity                  1.00000       0.00000      1.0000          1.0000
## Pos Pred Value                   NaN       0.03824         NaN             NaN
## Neg Pred Value               0.90294           NaN      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.03824      0.0000          0.0000
## Detection Prevalence         0.00000       1.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n5_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY        396    156  489     1063   578   608  790
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ           0      0    0        0     0     0    0
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.0382          
##                  95% CI : (0.0326, 0.0446)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       1.00000      0.0000          0.0000
## Specificity                  1.00000       0.00000      1.0000          1.0000
## Pos Pred Value                   NaN       0.03824         NaN             NaN
## Neg Pred Value               0.90294           NaN      0.8801          0.7395
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.03824      0.0000          0.0000
## Detection Prevalence         0.00000       1.00000      0.0000          0.0000
## Balanced Accuracy            0.50000       0.50000      0.5000          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.0000        0.000      0.0000
## Specificity                1.0000        1.000      1.0000
## Pos Pred Value                NaN          NaN         NaN
## Neg Pred Value             0.8583        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.0000        0.000      0.0000
## Detection Prevalence       0.0000        0.000      0.0000
## Balanced Accuracy          0.5000        0.500      0.5000
db_tda_pc_5.50.5_n5_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##     0.03823529     0.00000000     0.03256139     0.04458199     0.26053922 
## AccuracyPValue  McnemarPValue 
##     1.00000000            NaN
db_tda_pc_5.50.5_n5_db_lr_cf0_ov_acc<-db_tda_pc_5.50.5_n5_db_lr_cf0$overall[1]
db_tda_pc_5.50.5_n5_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value
## Class: BARBUNYA           0           1            NaN      0.9029412
## Class: BOMBAY             1           0     0.03823529            NaN
## Class: CALI               0           1            NaN      0.8801471
## Class: DERMASON           0           1            NaN      0.7394608
## Class: HOROZ              0           1            NaN      0.8583333
## Class: SEKER              0           1            NaN      0.8509804
## Class: SIRA               0           1            NaN      0.8063725
##                  Precision Recall         F1 Prevalence Detection Rate
## Class: BARBUNYA         NA      0         NA 0.09705882     0.00000000
## Class: BOMBAY   0.03823529      1 0.07365439 0.03823529     0.03823529
## Class: CALI             NA      0         NA 0.11985294     0.00000000
## Class: DERMASON         NA      0         NA 0.26053922     0.00000000
## Class: HOROZ            NA      0         NA 0.14166667     0.00000000
## Class: SEKER            NA      0         NA 0.14901961     0.00000000
## Class: SIRA             NA      0         NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA                    0               0.5
## Class: BOMBAY                      1               0.5
## Class: CALI                        0               0.5
## Class: DERMASON                    0               0.5
## Class: HOROZ                       0               0.5
## Class: SEKER                       0               0.5
## Class: SIRA                        0               0.5
db_tda_pc_5.50.5_n5_db_lr_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n5_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted LR vs. tda-assisted LR classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_lr_n5_3_fold<-(db_lr_fit_re - db_tda_pc_5.50.5_n5_lr_fit_re)
diff_drybean_tda_pca_5.50.5_lr_n5_3_fold
##      Accuracy
## 1          NA
## 2 -0.07960982
## 3 -0.07617249
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n5_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n5_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n5_3_fold
## $probLeft
## [1] NA
## 
## $probRope
## [1] NA
## 
## $probRight
## [1] NA
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n5_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n5_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n5_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n5_3_fold_odds.left
## [1] NA
# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_lr.n5_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n5_3_fold),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_lr.n5_3_fold

# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n5_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n5_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n5_3_fold
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_lr_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr_n5_3_fold))
#bf_tda_pca_5.50.5_lr.n5_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr_n5_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_lr_n5_3_fold)
## t = -45.321, df = 1, p-value = 0.01404
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.09972885 -0.05605346
## sample estimates:
##   mean of x 
## -0.07789115
### Test set diff
diff_drybean_tda_pca_5.50.5_lr.n5_test<-(db_lr_cf_ov_acc - db_tda_pc_5.50.5_n5_db_lr_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_lr.n5_test
##  Accuracy 
## 0.8818627
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_lr.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n5_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_lr.n5_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_lr.n5_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_lr.n5_test$probLeft/bst_dbf_db_tda_pca_5.50.5_lr.n5_test$probRight
bst_dbf_db_tda_pca_5.50.5_lr.n5_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_lr.n5_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n5_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_lr.n5_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1602
## 
## $winRight
## [1] 0.8398
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_lr.n5_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n5_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_lr.n5_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_lr.n5_test)))

#BayesFactor
#bf_tda_pca_5.50.5_lr.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_lr.n5_test)) #bf_tda_pca_5.50.5_lr.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_lr.n5_test))


##With TDA KDE filter 5 intervals, 50% overlap, 5 bins 
##Node1


DryBean_TDA_KDE_5.50.5_n1_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_kde_dry_bean_dataset_5.50.5.n1.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  126 (102 variable)
## initial  value 10988.554612 
## iter  10 value 6837.690948
## iter  20 value 5197.858707
## iter  30 value 4209.312091
## iter  40 value 1588.101774
## iter  50 value 900.856802
## iter  60 value 841.534537
## iter  70 value 818.666032
## iter  80 value 807.255799
## iter  90 value 795.197426
## iter 100 value 786.705103
## final  value 786.705103 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10988.554612 
## iter  10 value 6837.691015
## iter  20 value 5197.860532
## iter  30 value 4209.563304
## iter  40 value 1702.766466
## iter  50 value 1108.844291
## iter  60 value 1039.176591
## iter  70 value 959.172242
## iter  80 value 922.085946
## iter  90 value 909.933562
## iter 100 value 906.120155
## final  value 906.120155 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10988.554612 
## iter  10 value 6837.690948
## iter  20 value 5197.858711
## iter  30 value 4209.312450
## iter  40 value 1586.117791
## iter  50 value 901.819532
## iter  60 value 843.439990
## iter  70 value 822.156196
## iter  80 value 812.558876
## iter  90 value 803.375843
## iter 100 value 797.650438
## final  value 797.650438 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10990.500522 
## iter  10 value 6893.066216
## iter  20 value 4959.773884
## iter  30 value 3973.003781
## iter  40 value 2414.116122
## iter  50 value 854.241676
## iter  60 value 803.884228
## iter  70 value 788.864232
## iter  80 value 775.103786
## iter  90 value 768.758086
## iter 100 value 762.963093
## final  value 762.963093 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10990.500522 
## iter  10 value 6893.066273
## iter  20 value 4959.775140
## iter  30 value 3973.123995
## iter  40 value 2500.463966
## iter  50 value 1109.926310
## iter  60 value 1058.351838
## iter  70 value 1014.756570
## iter  80 value 974.159128
## iter  90 value 961.379358
## iter 100 value 941.668954
## final  value 941.668954 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10990.500522 
## iter  10 value 6893.066216
## iter  20 value 4959.773878
## iter  30 value 3973.002791
## iter  40 value 2414.278588
## iter  50 value 854.954671
## iter  60 value 805.558223
## iter  70 value 791.551980
## iter  80 value 779.816334
## iter  90 value 775.106310
## iter 100 value 770.972937
## final  value 770.972937 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10996.338252 
## iter  10 value 6734.360911
## iter  20 value 4820.347477
## iter  30 value 4160.737233
## iter  40 value 1881.503611
## iter  50 value 836.139438
## iter  60 value 776.249227
## iter  70 value 758.117590
## iter  80 value 744.524377
## iter  90 value 737.976497
## iter 100 value 731.936459
## final  value 731.936459 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10996.338252 
## iter  10 value 6734.361028
## iter  20 value 4820.349545
## iter  30 value 4160.780152
## iter  40 value 1930.494559
## iter  50 value 1064.628679
## iter  60 value 978.157816
## iter  70 value 906.251174
## iter  80 value 862.279139
## iter  90 value 853.042726
## iter 100 value 848.223491
## final  value 848.223491 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 10996.338252 
## iter  10 value 6734.360911
## iter  20 value 4820.347486
## iter  30 value 4160.737667
## iter  40 value 1881.554777
## iter  50 value 852.516910
## iter  60 value 781.564651
## iter  70 value 763.695852
## iter  80 value 751.532314
## iter  90 value 745.735728
## iter 100 value 741.850397
## final  value 741.850397 
## stopped after 100 iterations
## # weights:  126 (102 variable)
## initial  value 16487.696693 
## iter  10 value 12073.546982
## iter  20 value 8017.939505
## iter  30 value 6790.979763
## iter  40 value 2670.846437
## iter  50 value 1325.405930
## iter  60 value 1232.715991
## iter  70 value 1210.574996
## iter  80 value 1186.961362
## iter  90 value 1177.774537
## iter 100 value 1172.339193
## final  value 1172.339193 
## stopped after 100 iterations
DryBean_TDA_KDE_5.50.5_n1_LrFit0
## Penalized Multinomial Regression 
## 
## 8473 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5647, 5648, 5651 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.9529095  0.9437453
##   1e-04  0.9532635  0.9441686
##   1e-01  0.9498405  0.9400844
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 1e-04.
DryBean_TDA_KDE_5.50.5_n1_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9518584 0.9424883    Fold2
## 2 0.9539986 0.9450324    Fold1
## 3 0.9539334 0.9449850    Fold3
nb_tda_kde_5.50.5_n1_lr_fit_re<-DryBean_TDA_KDE_5.50.5_n1_LrFit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n1_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##          (Intercept)        Area   Perimeter MajorAxisLength MinorAxisLength
## BOMBAY     11.036690 0.003539723 -0.12632891       0.4795659       1.1733521
## CALI       34.788467 0.003118151 -0.18248782       1.9035078       2.5206426
## DERMASON   43.062469 0.007187764  0.07702549       1.3461987       1.7237278
## HOROZ       2.790146 0.007612553  0.08490787       2.1186404       4.0792536
## SEKER      -2.394236 0.004046423  0.17706303       0.6937010       0.1230065
## SIRA       42.315250 0.004875151 -0.13696761       2.1870636       2.3826037
##          AspectRation Eccentricity   ConvexArea EquivDiameter     Extent
## BOMBAY      31.261544     19.68799 -0.001477253     -2.001664 -0.6262265
## CALI       -67.559062    108.02139 -0.003336913     -3.652179  4.2446686
## DERMASON   -36.404278     92.96568 -0.004909342     -4.561369 -9.9686716
## HOROZ        2.212735    100.14038 -0.006732933     -6.688530 -3.0552608
## SEKER      -49.458266     11.38793 -0.003534883     -1.888427 -5.9326146
## SIRA       -85.909361    121.14643 -0.005221789     -4.280277 -5.8219881
##           Solidity  roundness Compactness ShapeFactor1 ShapeFactor2
## BOMBAY    11.37351   9.612811    4.258541    0.2795171  0.003449938
## CALI      39.29064 -50.104213   -1.844882    1.0517992  0.068985682
## DERMASON  39.71568  73.623679   27.991769    0.6917159  0.169481021
## HOROZ     28.47648  61.729422  -34.797416    0.9648725 -0.361563728
## SEKER    -22.21867 117.497351   25.948910   -0.8604716  0.069155764
## SIRA      35.06845 -29.357783   31.166003   -0.8037414 -0.154315025
##          ShapeFactor3 ShapeFactor4
## BOMBAY      -1.002353    10.783763
## CALI       -40.888530    -6.805262
## DERMASON     1.393097    21.077324
## HOROZ      -73.224152    -9.965704
## SEKER       47.274995     2.244941
## SIRA         1.318080    15.599805
## 
## Std. Errors:
##           (Intercept)         Area    Perimeter MajorAxisLength MinorAxisLength
## BOMBAY   9.318041e-09 0.0002152853 1.846957e-05    2.495589e-05    0.0000130445
## CALI     7.659164e-06 0.0002909245 1.078672e-03    2.538639e-03    0.0031595386
## DERMASON 1.096565e-05 0.0012649178 3.746415e-03    1.443026e-03    0.0014920154
## HOROZ    3.134598e-06 0.0003709058 1.834321e-03    7.222759e-04    0.0006482841
## SEKER    4.817368e-06 0.0007388128 2.477149e-03    6.015784e-04    0.0006498713
## SIRA     6.819379e-06 0.0004611934 2.160322e-03    2.686613e-03    0.0026962499
##          AspectRation Eccentricity   ConvexArea EquivDiameter       Extent
## BOMBAY   1.572731e-07 6.282629e-08 0.0002124159  2.092990e-06 1.922534e-08
## CALI     2.323675e-05 3.697050e-06 0.0002885588  1.122317e-03 7.379594e-06
## DERMASON 1.661914e-05 6.857915e-06 0.0012915125  1.202551e-03 9.028714e-06
## HOROZ    7.984153e-06 2.692018e-06 0.0003697209  4.803448e-04 2.688085e-06
## SEKER    5.315782e-06 3.022554e-06 0.0007476451  6.445913e-04 3.759532e-06
## SIRA     2.700959e-05 7.505602e-06 0.0004665447  8.683308e-04 7.475066e-06
##              Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## BOMBAY   1.277472e-08 3.966882e-08 3.069586e-08 2.490169e-10 1.395315e-10
## CALI     7.550245e-06 1.237620e-05 1.369620e-05 1.827909e-08 5.216299e-08
## DERMASON 1.086189e-05 1.075051e-05 1.107203e-05 1.084990e-07 5.167712e-08
## HOROZ    3.071049e-06 3.061709e-06 2.914602e-06 3.411292e-08 7.489913e-09
## SEKER    4.757292e-06 4.203744e-06 4.510602e-06 4.034340e-08 1.871658e-08
## SIRA     6.707306e-06 1.038645e-05 1.307908e-05 4.420472e-08 6.234767e-08
##          ShapeFactor3 ShapeFactor4
## BOMBAY   5.580891e-08 9.172119e-09
## CALI     1.658362e-05 7.548106e-06
## DERMASON 1.138473e-05 1.094934e-05
## HOROZ    2.891440e-06 3.127780e-06
## SEKER    4.207713e-06 4.816465e-06
## SIRA     1.680679e-05 6.876640e-06
## 
## Residual Deviance: 2344.678 
## AIC: 2548.678
vip(DryBean_TDA_KDE_5.50.5_n1_LrFit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n1_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n1_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n1_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n1_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
nb_tda_kde_5.50.5_n1_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   15        0     0     6    2
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           18      0  458        0    10     0    2
##   DERMASON        0      0    0      893     6     7   41
##   HOROZ           3      0    8        1   553     0   11
##   SEKER           3      0    1       23     0   576    6
##   SIRA            8      0    7      146     9    19  728
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9137          
##                  95% CI : (0.9047, 0.9222)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8959          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000      0.9366          0.8401
## Specificity                  0.99376       1.00000      0.9916          0.9821
## Pos Pred Value               0.94057       1.00000      0.9385          0.9430
## Neg Pred Value               0.99133       1.00000      0.9914          0.9457
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.03824      0.1123          0.2189
## Detection Prevalence         0.09485       0.03824      0.1196          0.2321
## Balanced Accuracy            0.95647       1.00000      0.9641          0.9111
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9567       0.9474      0.9215
## Specificity                0.9934       0.9905      0.9426
## Pos Pred Value             0.9601       0.9458      0.7939
## Neg Pred Value             0.9929       0.9908      0.9804
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1355       0.1412      0.1784
## Detection Prevalence       0.1412       0.1493      0.2248
## Balanced Accuracy          0.9751       0.9689      0.9320
nb_tda_kde_5.50.5_n1_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   15        0     0     6    2
##   BOMBAY          0    156    0        0     0     0    0
##   CALI           18      0  458        0    10     0    2
##   DERMASON        0      0    0      893     6     7   41
##   HOROZ           3      0    8        1   553     0   11
##   SEKER           3      0    1       23     0   576    6
##   SIRA            8      0    7      146     9    19  728
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9137          
##                  95% CI : (0.9047, 0.9222)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8959          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000      0.9366          0.8401
## Specificity                  0.99376       1.00000      0.9916          0.9821
## Pos Pred Value               0.94057       1.00000      0.9385          0.9430
## Neg Pred Value               0.99133       1.00000      0.9914          0.9457
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.03824      0.1123          0.2189
## Detection Prevalence         0.09485       0.03824      0.1196          0.2321
## Balanced Accuracy            0.95647       1.00000      0.9641          0.9111
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9567       0.9474      0.9215
## Specificity                0.9934       0.9905      0.9426
## Pos Pred Value             0.9601       0.9458      0.7939
## Neg Pred Value             0.9929       0.9908      0.9804
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1355       0.1412      0.1784
## Detection Prevalence       0.1412       0.1493      0.2248
## Balanced Accuracy          0.9751       0.9689      0.9320
nb_tda_kde_5.50.5_n1_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.9137255      0.8958994      0.9046885      0.9221647      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
nb_tda_kde_5.50.5_n1_db_lr_cf0_ov_acc<-nb_tda_kde_5.50.5_n1_db_lr_cf0$overall[1]
nb_tda_kde_5.50.5_n1_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9191919   0.9937568      0.9405685      0.9913350 0.9405685
## Class: BOMBAY     1.0000000   1.0000000      1.0000000      1.0000000 1.0000000
## Class: CALI       0.9366053   0.9916458      0.9385246      0.9913697 0.9385246
## Class: DERMASON   0.8400753   0.9821014      0.9429778      0.9457389 0.9429778
## Class: HOROZ      0.9567474   0.9934323      0.9600694      0.9928653 0.9600694
## Class: SEKER      0.9473684   0.9904954      0.9458128      0.9907808 0.9458128
## Class: SIRA       0.9215190   0.9425532      0.7938931      0.9803984 0.7938931
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9191919 0.9297573 0.09705882     0.08921569
## Class: BOMBAY   1.0000000 1.0000000 0.03823529     0.03823529
## Class: CALI     0.9366053 0.9375640 0.11985294     0.11225490
## Class: DERMASON 0.8400753 0.8885572 0.26053922     0.21887255
## Class: HOROZ    0.9567474 0.9584055 0.14166667     0.13553922
## Class: SEKER    0.9473684 0.9465900 0.14901961     0.14117647
## Class: SIRA     0.9215190 0.8529584 0.19362745     0.17843137
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09485294         0.9564744
## Class: BOMBAY             0.03823529         1.0000000
## Class: CALI               0.11960784         0.9641255
## Class: DERMASON           0.23210784         0.9110883
## Class: HOROZ              0.14117647         0.9750899
## Class: SEKER              0.14926471         0.9689319
## Class: SIRA               0.22475490         0.9320361
nb_tda_kde_5.50.5_n1_db_lr_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n1_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_lr_n1_3_fold<-(db_lr_fit_re - nb_tda_kde_5.50.5_n1_lr_fit_re)
diff_drybean_tda_kde_5.50.5_lr_n1_3_fold
##      Accuracy
## 1 -0.01546042
## 2 -0.03360840
## 3 -0.03010587
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n1_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n1_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n1_3_fold_odds.left<-bst_tda_kde_5.50.5_lr.n1_3_fold$probLeft/bst_tda_kde_5.50.5_lr.n1_3_fold$probRight
bst_tda_kde_5.50.5_lr.n1_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n1_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n1_3_fold
## $winLeft
## [1] 0.9617667
## 
## $winRope
## [1] 0.03823333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n1_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n1_3_fold
## $left
## [1] 0.9374152
## 
## $rope
## [1] 0.04772251
## 
## $right
## [1] 0.01486228
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_lr_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr_n1_3_fold))
#bf_tda_kde_5.50.5_lr.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_lr_n1_3_fold)
## t = -4.7481, df = 2, p-value = 0.04161
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.050307031 -0.002476099
## sample estimates:
##   mean of x 
## -0.02639156
### Test set diff
diff_drybean_tda_kde_5.50.5_lr.n1_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n1_db_lr_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_lr.n1_test
##   Accuracy 
## 0.01323529
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n1_test),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n1_test_odds.left<-bst_tda_kde_5.50.5_lr.n1_test$probLeft/bst_tda_kde_5.50.5_lr.n1_test$probRight
bst_tda_kde_5.50.5_lr.n1_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n1_test),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.4588667
## 
## $winRight
## [1] 0.5411333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n1_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_lr.n1_test)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr.n1_test)) #bf_tda_pca_5.50.5_lr.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n1_test))


##With TDA KDE filter 5 intervals, 50% overlap, 5 bins 
##Node2

DryBean_TDA_KDE_5.50.5_n2_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_kde_dry_bean_dataset_5.50.5.n2.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  108 (85 variable)
## initial  value 9055.552357 
## iter  10 value 4841.754969
## iter  20 value 3613.144007
## iter  30 value 2045.358066
## iter  40 value 956.594837
## iter  50 value 904.920532
## iter  60 value 879.362864
## iter  70 value 869.733222
## iter  80 value 859.111981
## iter  90 value 852.150184
## iter 100 value 846.790046
## final  value 846.790046 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9055.552357 
## iter  10 value 4841.755378
## iter  20 value 3613.149625
## iter  30 value 2198.848825
## iter  40 value 1099.427381
## iter  50 value 976.483536
## iter  60 value 951.343313
## iter  70 value 939.587329
## iter  80 value 937.307777
## iter  90 value 935.895627
## iter 100 value 935.701305
## final  value 935.701305 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9055.552357 
## iter  10 value 4841.754969
## iter  20 value 3613.144047
## iter  30 value 2045.473504
## iter  40 value 957.004476
## iter  50 value 905.809522
## iter  60 value 881.449094
## iter  70 value 872.726143
## iter  80 value 863.761529
## iter  90 value 858.221546
## iter 100 value 854.288928
## final  value 854.288928 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9059.135876 
## iter  10 value 4797.864962
## iter  20 value 3239.933230
## iter  30 value 2246.095003
## iter  40 value 868.319317
## iter  50 value 813.916795
## iter  60 value 790.717458
## iter  70 value 781.747402
## iter  80 value 777.729583
## iter  90 value 771.840791
## iter 100 value 769.585146
## final  value 769.585146 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9059.135876 
## iter  10 value 4797.865380
## iter  20 value 3239.941531
## iter  30 value 1863.032982
## iter  40 value 996.341382
## iter  50 value 909.554614
## iter  60 value 885.764824
## iter  70 value 874.365007
## iter  80 value 872.518879
## iter  90 value 871.941056
## iter 100 value 871.876120
## final  value 871.876120 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9059.135876 
## iter  10 value 4797.864962
## iter  20 value 3239.933263
## iter  30 value 2246.223632
## iter  40 value 868.859521
## iter  50 value 815.125963
## iter  60 value 793.398271
## iter  70 value 785.501274
## iter  80 value 782.139346
## iter  90 value 777.437977
## iter 100 value 775.744918
## final  value 775.744918 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9055.552357 
## iter  10 value 4885.533572
## iter  20 value 3254.751513
## iter  30 value 1591.150344
## iter  40 value 892.885881
## iter  50 value 861.906289
## iter  60 value 843.176159
## iter  70 value 835.597137
## iter  80 value 828.356986
## iter  90 value 822.923143
## iter 100 value 818.553246
## final  value 818.553246 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9055.552357 
## iter  10 value 4885.534002
## iter  20 value 3254.770451
## iter  30 value 1662.261780
## iter  40 value 1034.946397
## iter  50 value 947.479698
## iter  60 value 923.997427
## iter  70 value 913.141775
## iter  80 value 911.564098
## iter  90 value 911.190393
## iter 100 value 911.115252
## final  value 911.115252 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 9055.552357 
## iter  10 value 4885.533572
## iter  20 value 3254.751552
## iter  30 value 1591.227782
## iter  40 value 893.443994
## iter  50 value 862.961428
## iter  60 value 845.394692
## iter  70 value 838.641043
## iter  80 value 832.569662
## iter  90 value 828.114443
## iter 100 value 824.957455
## final  value 824.957455 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 13585.120296 
## iter  10 value 7273.804421
## iter  20 value 4067.771438
## iter  30 value 2518.237578
## iter  40 value 1337.595482
## iter  50 value 1286.454409
## iter  60 value 1262.308765
## iter  70 value 1249.188251
## iter  80 value 1243.729097
## iter  90 value 1238.157229
## iter 100 value 1233.653115
## final  value 1233.653115 
## stopped after 100 iterations
DryBean_TDA_KDE_5.50.5_n2_LrFit0
## Penalized Multinomial Regression 
## 
## 7582 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5054, 5056, 5054 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.9460565  0.9310555
##   1e-04  0.9465841  0.9317214
##   1e-01  0.9423631  0.9263261
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 1e-04.
DryBean_TDA_KDE_5.50.5_n2_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9469517 0.9322544    Fold2
## 2 0.9465981 0.9317144    Fold1
## 3 0.9462025 0.9311955    Fold3
nb_tda_kde_5.50.5_n2_lr_fit_re<-DryBean_TDA_KDE_5.50.5_n2_LrFit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n2_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##          (Intercept)        Area   Perimeter MajorAxisLength MinorAxisLength
## CALI        19.05626 0.002972756 -0.18443391      2.37346646       2.8822418
## DERMASON    19.05755 0.001097468 -0.04111903      1.44714806       1.4100183
## HOROZ      -10.81143 0.007910822  0.04351992      2.18621975       4.6380524
## SEKER      -10.40457 0.004020212  0.09668340     -0.09602789      -0.6554541
## SIRA        57.72405 0.003361244 -0.43402571      2.23762796       2.9161683
##          AspectRation Eccentricity   ConvexArea EquivDiameter     Extent
## CALI        -52.31975     32.05919 -0.003380958    -4.4444531   5.786511
## DERMASON    -33.44919     23.04210 -0.002968513    -2.5642276 -15.821565
## HOROZ        47.49808     67.61948 -0.006171255    -7.4157268  -5.292679
## SEKER        13.55370    -80.56346 -0.003778468     0.1267185 -10.843799
## SIRA        -42.48669    101.05967 -0.004301807    -3.5098273  -8.516168
##           Solidity  roundness Compactness ShapeFactor1 ShapeFactor2
## CALI      20.81700  -43.59746    12.48273   0.31568270  0.124237561
## DERMASON   3.88883   37.49429    14.10902   0.09331638  0.008089463
## HOROZ     34.95519   51.96063   -36.60310   0.06427325 -0.404167894
## SEKER    -23.84906   85.71036    13.52889  -0.28296050  0.176556626
## SIRA      29.19704 -159.14051    39.83074   1.05375697  0.348855747
##          ShapeFactor3 ShapeFactor4
## CALI         2.410771     5.178713
## DERMASON     1.157556     7.848230
## HOROZ      -56.880568    -6.868662
## SEKER       43.828024     7.621281
## SIRA         7.794554    20.303658
## 
## Std. Errors:
##           (Intercept)         Area   Perimeter MajorAxisLength MinorAxisLength
## CALI     4.128369e-06 0.0005467353 0.001833236    0.0008854458    0.0006586432
## DERMASON 9.550503e-06 0.0009669833 0.003205864    0.0018168526    0.0020248641
## HOROZ    4.593080e-06 0.0006432548 0.002013569    0.0007996313    0.0008381670
## SEKER    8.003805e-06 0.0012080434 0.003023476    0.0006288964    0.0014312720
## SIRA     8.267572e-06 0.0006674013 0.002573272    0.0034336249    0.0034765827
##          AspectRation Eccentricity   ConvexArea EquivDiameter       Extent
## CALI     7.901234e-06 3.195765e-06 0.0005438585  0.0005760083 3.370255e-06
## DERMASON 2.059419e-05 7.785112e-06 0.0009819891  0.0010286804 9.073847e-06
## HOROZ    6.997347e-06 3.005486e-06 0.0006384838  0.0006917930 4.184887e-06
## SEKER    6.417475e-06 3.000065e-06 0.0012109828  0.0010104162 6.925234e-06
## SIRA     3.648878e-05 1.112685e-05 0.0006716057  0.0010123793 9.787766e-06
##              Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## CALI     4.075204e-06 4.108339e-06 3.871547e-06 3.474644e-08 1.297776e-08
## DERMASON 9.431160e-06 1.012105e-05 1.269471e-05 8.634762e-08 6.658832e-08
## HOROZ    4.504002e-06 4.718564e-06 4.658357e-06 3.408746e-08 1.570244e-08
## SEKER    7.925962e-06 7.911783e-06 8.838574e-06 5.982171e-08 3.533132e-08
## SIRA     8.143670e-06 1.349461e-05 1.756611e-05 5.156651e-08 9.225182e-08
##          ShapeFactor3 ShapeFactor4
## CALI     3.803370e-06 4.101324e-06
## DERMASON 1.509421e-05 9.554445e-06
## HOROZ    4.591064e-06 4.524234e-06
## SEKER    9.207185e-06 8.000819e-06
## SIRA     2.315503e-05 8.314071e-06
## 
## Residual Deviance: 2467.306 
## AIC: 2637.306
vip(DryBean_TDA_KDE_5.50.5_n2_LrFit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n2_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n2_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n2_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n2_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n2_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   15        1     1     7    5
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           19      0  461        0    18     2    1
##   DERMASON        0      0    0      946     6     7   74
##   HOROZ           3    156    6        5   543     0    7
##   SEKER           2      0    1       24     0   574    2
##   SIRA            8      0    6       87    10    18  701
## 
## Overall Statistics
##                                           
##                Accuracy : 0.8797          
##                  95% CI : (0.8693, 0.8895)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8539          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       0.00000      0.9427          0.8899
## Specificity                  0.99213       1.00000      0.9889          0.9712
## Pos Pred Value               0.92621           NaN      0.9202          0.9158
## Neg Pred Value               0.99132       0.96176      0.9922          0.9616
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.00000      0.1130          0.2319
## Detection Prevalence         0.09632       0.00000      0.1228          0.2532
## Balanced Accuracy            0.95566       0.50000      0.9658          0.9305
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9394       0.9441      0.8873
## Specificity                0.9495       0.9916      0.9608
## Pos Pred Value             0.7542       0.9519      0.8446
## Neg Pred Value             0.9896       0.9902      0.9726
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1331       0.1407      0.1718
## Detection Prevalence       0.1765       0.1478      0.2034
## Balanced Accuracy          0.9445       0.9679      0.9241
nb_tda_kde_5.50.5_n2_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   15        1     1     7    5
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           19      0  461        0    18     2    1
##   DERMASON        0      0    0      946     6     7   74
##   HOROZ           3    156    6        5   543     0    7
##   SEKER           2      0    1       24     0   574    2
##   SIRA            8      0    6       87    10    18  701
## 
## Overall Statistics
##                                           
##                Accuracy : 0.8797          
##                  95% CI : (0.8693, 0.8895)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8539          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       0.00000      0.9427          0.8899
## Specificity                  0.99213       1.00000      0.9889          0.9712
## Pos Pred Value               0.92621           NaN      0.9202          0.9158
## Neg Pred Value               0.99132       0.96176      0.9922          0.9616
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.08922       0.00000      0.1130          0.2319
## Detection Prevalence         0.09632       0.00000      0.1228          0.2532
## Balanced Accuracy            0.95566       0.50000      0.9658          0.9305
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9394       0.9441      0.8873
## Specificity                0.9495       0.9916      0.9608
## Pos Pred Value             0.7542       0.9519      0.8446
## Neg Pred Value             0.9896       0.9902      0.9726
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1331       0.1407      0.1718
## Detection Prevalence       0.1765       0.1478      0.2034
## Balanced Accuracy          0.9445       0.9679      0.9241
nb_tda_kde_5.50.5_n2_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.8796569      0.8538735      0.8692781      0.8894885      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
nb_tda_kde_5.50.5_n2_db_lr_cf0_ov_acc<-nb_tda_kde_5.50.5_n2_db_lr_cf0$overall[1]
nb_tda_kde_5.50.5_n2_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9191919   0.9921281      0.9262087      0.9913209 0.9262087
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.9427403   0.9888610      0.9201597      0.9921766 0.9201597
## Class: DERMASON   0.8899341   0.9711634      0.9157793      0.9616016 0.9157793
## Class: HOROZ      0.9394464   0.9494575      0.7541667      0.9895833 0.7541667
## Class: SEKER      0.9440789   0.9916475      0.9519071      0.9902215 0.9519071
## Class: SIRA       0.8873418   0.9607903      0.8445783      0.9726154 0.8445783
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9191919 0.9226869 0.09705882     0.08921569
## Class: BOMBAY   0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.9427403 0.9313131 0.11985294     0.11299020
## Class: DERMASON 0.8899341 0.9026718 0.26053922     0.23186275
## Class: HOROZ    0.9394464 0.8366718 0.14166667     0.13308824
## Class: SEKER    0.9440789 0.9479769 0.14901961     0.14068627
## Class: SIRA     0.8873418 0.8654321 0.19362745     0.17181373
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09632353         0.9556600
## Class: BOMBAY             0.00000000         0.5000000
## Class: CALI               0.12279412         0.9658007
## Class: DERMASON           0.25318627         0.9305488
## Class: HOROZ              0.17647059         0.9444519
## Class: SEKER              0.14779412         0.9678632
## Class: SIRA               0.20343137         0.9240660
nb_tda_kde_5.50.5_n2_db_lr_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n2_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_lr_n2_3_fold<-(db_lr_fit_re - nb_tda_kde_5.50.5_n2_lr_fit_re)
diff_drybean_tda_kde_5.50.5_lr_n2_3_fold
##      Accuracy
## 1 -0.01055372
## 2 -0.02620792
## 3 -0.02237502
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n2_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n2_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n2_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n2_3_fold_odds.left<-bst_tda_kde_5.50.5_lr.n2_3_fold$probLeft/bst_tda_kde_5.50.5_lr.n2_3_fold$probRight
bst_tda_kde_5.50.5_lr.n2_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n2_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n2_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n2_3_fold
## $winLeft
## [1] 0.9654
## 
## $winRope
## [1] 0.0346
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n2_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n2_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n2_3_fold
## $left
## [1] 0.8919397
## 
## $rope
## [1] 0.0920987
## 
## $right
## [1] 0.01596165
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_lr_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr_n2_3_fold))
#bf_tda_kde_5.50.5_lr.n2_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n2_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_lr_n2_3_fold)
## t = -4.1843, df = 2, p-value = 0.05265
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.039982142  0.000557704
## sample estimates:
##   mean of x 
## -0.01971222
### Test set diff
diff_drybean_tda_kde_5.50.5_lr.n2_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n2_db_lr_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_lr.n2_test
##   Accuracy 
## 0.04730392
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n2_test),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n2_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n2_test_odds.left<-bst_tda_kde_5.50.5_lr.n2_test$probLeft/bst_tda_kde_5.50.5_lr.n2_test$probRight
bst_tda_kde_5.50.5_lr.n2_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n2_test),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1594333
## 
## $winRight
## [1] 0.8405667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n2_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n2_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n2_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_lr.n2_test)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr.n2_test)) #bf_tda_pca_5.50.5_lr.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n2_test))

##Node3

DryBean_TDA_KDE_5.50.5_n3_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_kde_dry_bean_dataset_5.50.5.n3.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1195.706177
## iter  20 value 1065.011946
## iter  30 value 814.441109
## iter  40 value 677.658815
## iter  50 value 662.806446
## iter  60 value 648.184375
## iter  70 value 643.981421
## iter  80 value 639.639125
## iter  90 value 638.095403
## iter 100 value 636.750714
## final  value 636.750714 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1195.707226
## iter  20 value 1065.019948
## iter  30 value 814.939777
## iter  40 value 697.797267
## iter  50 value 689.623825
## iter  60 value 686.109002
## iter  70 value 685.915664
## iter  80 value 685.845849
## iter  90 value 685.759311
## iter 100 value 685.737093
## final  value 685.737093 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1195.706178
## iter  20 value 1065.011956
## iter  30 value 814.438714
## iter  40 value 677.854365
## iter  50 value 663.568928
## iter  60 value 650.647272
## iter  70 value 647.468128
## iter  80 value 644.450875
## iter  90 value 643.537196
## iter 100 value 642.575933
## final  value 642.575933 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1288.032700
## iter  20 value 1181.673467
## iter  30 value 852.243571
## iter  40 value 647.917367
## iter  50 value 627.336754
## iter  60 value 610.211801
## iter  70 value 605.181875
## iter  80 value 598.132085
## iter  90 value 595.245127
## iter 100 value 593.634561
## final  value 593.634561 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1288.034040
## iter  20 value 1181.682717
## iter  30 value 854.522248
## iter  40 value 666.936505
## iter  50 value 656.067444
## iter  60 value 654.607663
## iter  70 value 654.250515
## iter  80 value 654.220792
## iter  90 value 654.214878
## iter 100 value 654.212551
## final  value 654.212551 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1288.032701
## iter  20 value 1181.673473
## iter  30 value 852.244633
## iter  40 value 648.056982
## iter  50 value 628.026888
## iter  60 value 612.862070
## iter  70 value 608.774775
## iter  80 value 603.982874
## iter  90 value 602.105647
## iter 100 value 601.205908
## final  value 601.205908 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1224.952831
## iter  20 value 1040.910707
## iter  30 value 852.827231
## iter  40 value 706.297630
## iter  50 value 687.906681
## iter  60 value 673.832971
## iter  70 value 667.871553
## iter  80 value 664.580538
## iter  90 value 663.005357
## iter 100 value 661.798833
## final  value 661.798833 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1224.954207
## iter  20 value 1040.918731
## iter  30 value 857.064042
## iter  40 value 732.568445
## iter  50 value 724.379975
## iter  60 value 722.151524
## iter  70 value 722.004282
## iter  80 value 721.989733
## iter  90 value 721.984724
## final  value 721.984035 
## converged
## # weights:  108 (85 variable)
## initial  value 4956.006692 
## iter  10 value 1224.952832
## iter  20 value 1040.910714
## iter  30 value 852.832432
## iter  40 value 706.522106
## iter  50 value 688.838648
## iter  60 value 676.257754
## iter  70 value 671.570896
## iter  80 value 669.340353
## iter  90 value 668.315497
## iter 100 value 667.567329
## final  value 667.567329 
## stopped after 100 iterations
## # weights:  108 (85 variable)
## initial  value 7434.010038 
## iter  10 value 1744.375409
## iter  20 value 1457.282547
## iter  30 value 1202.391812
## iter  40 value 1019.568915
## iter  50 value 992.648147
## iter  60 value 975.688104
## iter  70 value 968.729438
## iter  80 value 964.879512
## iter  90 value 963.568311
## iter 100 value 962.064725
## final  value 962.064725 
## stopped after 100 iterations
DryBean_TDA_KDE_5.50.5_n3_LrFit0
## Penalized Multinomial Regression 
## 
## 4149 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 2766, 2766, 2766 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.9156423  0.8726380
##   1e-04  0.9166064  0.8740869
##   1e-01  0.9103398  0.8646375
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 1e-04.
DryBean_TDA_KDE_5.50.5_n3_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9016631 0.8515150    Fold2
## 2 0.9219089 0.8820536    Fold1
## 3 0.9262473 0.8886922    Fold3
nb_tda_kde_5.50.5_n3_lr_fit_re<-DryBean_TDA_KDE_5.50.5_n2_LrFit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n3_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##          (Intercept)         Area   Perimeter MajorAxisLength MinorAxisLength
## CALI      -0.4547832 -0.006686471 -0.39806753       0.3612705       0.2612429
## DERMASON -16.7215241  0.006064208 -0.05971107       1.3504345       2.1952024
## HOROZ      5.6639707  0.013078735 -0.03916328       2.9179190       4.4775828
## SEKER    -11.9129808  0.017104902 -0.01422559      -0.4950714      -2.3118436
## SIRA      29.3743719  0.008363490 -0.19841544       2.2305729       3.5932557
##          AspectRation Eccentricity   ConvexArea EquivDiameter     Extent
## CALI        -2.451518     2.900569  0.006791898     0.6637701  -2.842236
## DERMASON    13.706092     9.165755 -0.010656888    -2.1505813 -26.463131
## HOROZ       16.749448    25.632926 -0.014498033    -7.0049110 -28.027500
## SEKER        1.381761  -159.864136 -0.019912647     3.3416649 -25.055253
## SIRA       -24.996908   107.326608 -0.009655952    -4.8362221 -18.055560
##             Solidity  roundness  Compactness ShapeFactor1 ShapeFactor2
## CALI      -0.5297753  -1.523597  -1.45190263 -0.002323848  -0.01460178
## DERMASON -29.6193325  31.511427 -17.38274531 -0.192787067  -0.15238350
## HOROZ      8.7620460  11.671619  -2.09244038  0.195824941  -0.09936350
## SEKER    -13.2561450   8.974547  33.31830763 -0.565351671   0.53249528
## SIRA      37.1121303 -55.086724  -0.08419269  0.614973150  -0.13565132
##          ShapeFactor3 ShapeFactor4
## CALI        -2.640433    -1.374212
## DERMASON   -17.956351   -14.237086
## HOROZ       -9.316437     3.918728
## SEKER       84.506756    17.114260
## SIRA       -35.926305     4.085741
## 
## Std. Errors:
##           (Intercept)        Area    Perimeter MajorAxisLength MinorAxisLength
## CALI     1.232850e-07 0.001457998 3.591229e-05    0.0000259726    3.868781e-05
## DERMASON 8.294525e-06 0.001774921 1.753566e-03    0.0031355712    3.219529e-03
## HOROZ    9.016115e-06 0.002046382 3.490464e-03    0.0012307570    8.479790e-04
## SEKER    7.967033e-06 0.002212789 2.624260e-03    0.0007050524    1.007435e-03
## SIRA     8.679085e-06 0.001689665 1.796470e-03    0.0030693032    3.321951e-03
##          AspectRation Eccentricity  ConvexArea EquivDiameter       Extent
## CALI     2.536659e-07 6.768354e-08 0.001444405  1.832427e-05 8.329950e-08
## DERMASON 3.420737e-05 1.243350e-05 0.001748342  8.880873e-04 8.903549e-06
## HOROZ    1.399760e-05 6.937918e-06 0.002009333  1.022886e-03 6.659905e-06
## SEKER    7.527676e-06 3.972568e-06 0.002190989  8.922566e-04 6.720800e-06
## SIRA     3.356080e-05 1.159736e-05 0.001663321  9.581201e-04 9.467200e-06
##              Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## CALI     1.758341e-07 2.972853e-07 2.238032e-07 3.540158e-10 1.068808e-09
## DERMASON 8.178422e-06 1.229758e-05 1.768299e-05 5.753505e-08 1.034629e-07
## HOROZ    8.862901e-06 7.702615e-06 7.193116e-06 9.638475e-08 2.317920e-08
## SEKER    7.905902e-06 8.242242e-06 7.827483e-06 7.213453e-08 3.345469e-08
## SIRA     8.565068e-06 1.264722e-05 1.824873e-05 4.898856e-08 1.041735e-07
##          ShapeFactor3 ShapeFactor4
## CALI     2.808193e-07 1.562857e-07
## DERMASON 2.374322e-05 8.305578e-06
## HOROZ    5.732084e-06 8.956815e-06
## SEKER    7.539217e-06 7.961519e-06
## SIRA     2.420716e-05 8.727594e-06
## 
## Residual Deviance: 1924.129 
## AIC: 2094.129
vip(DryBean_TDA_KDE_5.50.5_n3_LrFit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n3_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n3_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n3_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n3_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n3_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      386      2  220        1    30    13   16
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            6    154  262        0     2    14    7
##   DERMASON        0      0    0      967     5    11   95
##   HOROZ           0      0    2        5   514     0   11
##   SEKER           0      0    0       25     0   560    2
##   SIRA            4      0    5       65    27    10  659
## 
## Overall Statistics
##                                           
##                Accuracy : 0.8206          
##                  95% CI : (0.8085, 0.8323)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.7827          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.97475       0.00000     0.53579          0.9097
## Specificity                  0.92345       1.00000     0.94904          0.9632
## Pos Pred Value               0.57784           NaN     0.58876          0.8970
## Neg Pred Value               0.99707       0.96176     0.93755          0.9680
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.09461       0.00000     0.06422          0.2370
## Detection Prevalence         0.16373       0.00000     0.10907          0.2642
## Balanced Accuracy            0.94910       0.50000     0.74241          0.9364
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.8893       0.9211      0.8342
## Specificity                0.9949       0.9922      0.9663
## Pos Pred Value             0.9662       0.9540      0.8558
## Neg Pred Value             0.9820       0.9863      0.9604
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1260       0.1373      0.1615
## Detection Prevalence       0.1304       0.1439      0.1887
## Balanced Accuracy          0.9421       0.9566      0.9002
nb_tda_kde_5.50.5_n3_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      386      2  220        1    30    13   16
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            6    154  262        0     2    14    7
##   DERMASON        0      0    0      967     5    11   95
##   HOROZ           0      0    2        5   514     0   11
##   SEKER           0      0    0       25     0   560    2
##   SIRA            4      0    5       65    27    10  659
## 
## Overall Statistics
##                                           
##                Accuracy : 0.8206          
##                  95% CI : (0.8085, 0.8323)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.7827          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.97475       0.00000     0.53579          0.9097
## Specificity                  0.92345       1.00000     0.94904          0.9632
## Pos Pred Value               0.57784           NaN     0.58876          0.8970
## Neg Pred Value               0.99707       0.96176     0.93755          0.9680
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.09461       0.00000     0.06422          0.2370
## Detection Prevalence         0.16373       0.00000     0.10907          0.2642
## Balanced Accuracy            0.94910       0.50000     0.74241          0.9364
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.8893       0.9211      0.8342
## Specificity                0.9949       0.9922      0.9663
## Pos Pred Value             0.9662       0.9540      0.8558
## Neg Pred Value             0.9820       0.9863      0.9604
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1260       0.1373      0.1615
## Detection Prevalence       0.1304       0.1439      0.1887
## Balanced Accuracy          0.9421       0.9566      0.9002
nb_tda_kde_5.50.5_n3_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.8205882      0.7827271      0.8084640      0.8322516      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
nb_tda_kde_5.50.5_n3_db_lr_cf0_ov_acc<-nb_tda_kde_5.50.5_n3_db_lr_cf0$overall[1]
nb_tda_kde_5.50.5_n3_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9747475   0.9234528      0.5778443      0.9970692 0.5778443
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.5357873   0.9490393      0.5887640      0.9375516 0.5887640
## Class: DERMASON   0.9096896   0.9632085      0.8970315      0.9680213 0.8970315
## Class: HOROZ      0.8892734   0.9948601      0.9661654      0.9819617 0.9661654
## Class: SEKER      0.9210526   0.9922235      0.9540034      0.9862582 0.9540034
## Class: SIRA       0.8341772   0.9662614      0.8558442      0.9604230 0.8558442
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9747475 0.7255639 0.09705882     0.09460784
## Class: BOMBAY   0.0000000        NA 0.03823529     0.00000000
## Class: CALI     0.5357873 0.5610278 0.11985294     0.06421569
## Class: DERMASON 0.9096896 0.9033162 0.26053922     0.23700980
## Class: HOROZ    0.8892734 0.9261261 0.14166667     0.12598039
## Class: SEKER    0.9210526 0.9372385 0.14901961     0.13725490
## Class: SIRA     0.8341772 0.8448718 0.19362745     0.16151961
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA            0.1637255         0.9491001
## Class: BOMBAY              0.0000000         0.5000000
## Class: CALI                0.1090686         0.7424133
## Class: DERMASON            0.2642157         0.9364490
## Class: HOROZ               0.1303922         0.9420667
## Class: SEKER               0.1438725         0.9566381
## Class: SIRA                0.1887255         0.9002193
nb_tda_kde_5.50.5_n3_db_lr_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n3_db_lr_cf0$byClass[5:7]


###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_lr_n3_3_fold<-(db_lr_fit_re - nb_tda_kde_5.50.5_n3_lr_fit_re)
diff_drybean_tda_kde_5.50.5_lr_n3_3_fold
##      Accuracy
## 1 -0.01055372
## 2 -0.02620792
## 3 -0.02237502
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n3_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n3_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n3_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n3_3_fold_odds.left<-bst_tda_kde_5.50.5_lr.n3_3_fold$probLeft/bst_tda_kde_5.50.5_lr.n3_3_fold$probRight
bst_tda_kde_5.50.5_lr.n3_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n3_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n3_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n3_3_fold
## $winLeft
## [1] 0.9626667
## 
## $winRope
## [1] 0.03733333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n3_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n3_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n3_3_fold
## $left
## [1] 0.8919397
## 
## $rope
## [1] 0.0920987
## 
## $right
## [1] 0.01596165
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_lr_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr_n3_3_fold))
#bf_tda_kde_5.50.5_lr.n3_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n3_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_lr_n3_3_fold)
## t = -4.1843, df = 2, p-value = 0.05265
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.039982142  0.000557704
## sample estimates:
##   mean of x 
## -0.01971222
### Test set diff
diff_drybean_tda_kde_5.50.5_lr.n3_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n3_db_lr_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_lr.n3_test
##  Accuracy 
## 0.1063725
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n3_test),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n3_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n3_test_odds.left<-bst_tda_kde_5.50.5_lr.n3_test$probLeft/bst_tda_kde_5.50.5_lr.n3_test$probRight
bst_tda_kde_5.50.5_lr.n3_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n2_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n2_test),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n2_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1600667
## 
## $winRight
## [1] 0.8399333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n3_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n3_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n3_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_lr.n3_test)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr.n3_test)) #bf_tda_pca_5.50.5_lr.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n3_test))

##Node4

DryBean_TDA_KDE_5.50.5_n4_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_kde_dry_bean_dataset_5.50.5.n4.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  72 (51 variable)
## initial  value 1871.497388 
## iter  10 value 930.739283
## iter  20 value 662.968620
## iter  30 value 562.949721
## iter  40 value 558.699683
## iter  50 value 555.577200
## iter  60 value 554.718693
## iter  70 value 554.250782
## iter  80 value 552.902177
## iter  90 value 549.601719
## iter 100 value 548.240729
## final  value 548.240729 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1871.497388 
## iter  10 value 930.743291
## iter  20 value 654.196441
## iter  30 value 572.553835
## iter  40 value 571.378512
## iter  50 value 570.549830
## final  value 570.549812 
## converged
## # weights:  72 (51 variable)
## initial  value 1871.497388 
## iter  10 value 930.739290
## iter  20 value 662.979260
## iter  30 value 563.002734
## iter  40 value 558.960867
## iter  50 value 556.147439
## iter  60 value 555.467776
## iter  70 value 555.153613
## iter  80 value 554.295152
## iter  90 value 553.568951
## iter 100 value 553.453282
## final  value 553.453282 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1870.111093 
## iter  10 value 1065.352490
## iter  20 value 638.738236
## iter  30 value 577.977780
## iter  40 value 569.061474
## iter  50 value 567.166938
## iter  60 value 565.350490
## iter  70 value 564.263113
## iter  80 value 562.412316
## iter  90 value 556.982392
## iter 100 value 554.327607
## final  value 554.327607 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1870.111093 
## iter  10 value 1065.363706
## iter  20 value 641.510031
## iter  30 value 585.575220
## iter  40 value 579.250476
## iter  50 value 579.209985
## iter  50 value 579.209981
## iter  50 value 579.209981
## final  value 579.209981 
## converged
## # weights:  72 (51 variable)
## initial  value 1870.111093 
## iter  10 value 1065.352500
## iter  20 value 638.737255
## iter  30 value 578.023357
## iter  40 value 569.271829
## iter  50 value 567.480117
## iter  60 value 566.147805
## iter  70 value 565.410862
## iter  80 value 564.552597
## iter  90 value 563.281814
## iter 100 value 563.098932
## final  value 563.098932 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1870.111093 
## iter  10 value 981.352959
## iter  20 value 624.524214
## iter  30 value 552.178909
## iter  40 value 544.090357
## iter  50 value 541.115949
## iter  60 value 540.345399
## iter  70 value 537.280546
## iter  80 value 533.336018
## iter  90 value 530.758399
## iter 100 value 529.299163
## final  value 529.299163 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 1870.111093 
## iter  10 value 981.356752
## iter  20 value 627.818755
## iter  30 value 563.383790
## iter  40 value 557.659687
## iter  50 value 557.157218
## final  value 557.157147 
## converged
## # weights:  72 (51 variable)
## initial  value 1870.111093 
## iter  10 value 981.352962
## iter  20 value 624.527971
## iter  30 value 552.262266
## iter  40 value 544.437907
## iter  50 value 541.613951
## iter  60 value 540.999614
## iter  70 value 539.130345
## iter  80 value 537.963976
## iter  90 value 537.341228
## iter 100 value 537.114039
## final  value 537.114039 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 2805.859787 
## iter  10 value 1225.844690
## iter  20 value 937.240180
## iter  30 value 843.229442
## iter  40 value 840.457792
## iter  50 value 836.485621
## iter  60 value 835.697501
## iter  70 value 833.187020
## iter  80 value 831.611471
## iter  90 value 831.283989
## iter 100 value 831.004772
## final  value 831.004772 
## stopped after 100 iterations
DryBean_TDA_KDE_5.50.5_n4_LrFit0
## Penalized Multinomial Regression 
## 
## 2024 samples
##   16 predictor
##    4 classes: 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1350, 1349, 1349 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.8246049  0.7074089
##   1e-04  0.8265787  0.7107460
##   1e-01  0.8241089  0.7053588
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 1e-04.
DryBean_TDA_KDE_5.50.5_n4_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.8459259 0.7397359    Fold2
## 2 0.8219585 0.7073075    Fold1
## 3 0.8118519 0.6851945    Fold3
nb_tda_kde_5.50.5_n4_lr_fit_re<-DryBean_TDA_KDE_5.50.5_n4_LrFit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n4_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##       (Intercept)         Area  Perimeter MajorAxisLength MinorAxisLength
## HOROZ    4.918146  0.002934240  0.0340530       1.2686575        1.890670
## SEKER   14.549214  0.016832642  0.2349197      -2.4124255       -4.111577
## SIRA    -7.805785 -0.003940889 -0.2229275       0.4346715        1.073650
##       AspectRation Eccentricity    ConvexArea EquivDiameter    Extent
## HOROZ    6.7976035     11.50951 -0.0014588191    -3.6236455 -2.121843
## SEKER   -0.4335939    -98.54492 -0.0127610504     4.3973803  6.895134
## SIRA   -20.8535497     45.79276  0.0001936836     0.7839925  8.060296
##         Solidity   roundness Compactness ShapeFactor1 ShapeFactor2 ShapeFactor3
## HOROZ  3.3594006    2.073014   0.9943935  0.132559269  -0.01506593    -2.774977
## SEKER  9.3460751   29.410410  43.6604763 -0.001089737   0.57677371    74.855437
## SIRA  -0.9901068 -129.867401 -22.4596521 -0.047209493  -0.28273544   -39.151457
##       ShapeFactor4
## HOROZ     2.594087
## SEKER    28.878592
## SIRA    -17.729635
## 
## Std. Errors:
##        (Intercept)         Area    Perimeter MajorAxisLength MinorAxisLength
## HOROZ 6.143166e-07 0.0031303830 0.0004471674    0.0001467535    1.793773e-05
## SEKER 1.719594e-05 0.0018293913 0.0059398483    0.0018238268    1.923942e-03
## SIRA  6.269943e-06 0.0008268668 0.0025157936    0.0011446475    5.332347e-04
##       AspectRation Eccentricity   ConvexArea EquivDiameter       Extent
## HOROZ 1.635039e-06 7.070129e-07 0.0030824329  5.833541e-05 2.617217e-07
## SEKER 1.926801e-05 9.285965e-06 0.0018674208  1.907210e-03 1.347836e-05
## SIRA  1.261138e-05 5.910277e-06 0.0008427697  6.885449e-04 4.678656e-06
##           Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## HOROZ 4.478714e-07 1.241902e-07 2.757438e-07 8.992689e-09 2.984888e-10
## SEKER 1.705822e-05 1.675000e-05 1.603274e-05 1.629743e-07 6.744617e-08
## SIRA  6.215288e-06 5.129600e-06 4.732800e-06 7.310132e-08 1.734507e-08
##       ShapeFactor3 ShapeFactor4
## HOROZ 5.632570e-08 5.071539e-07
## SEKER 1.484440e-05 1.719834e-05
## SIRA  3.972577e-06 6.254176e-06
## 
## Residual Deviance: 1662.01 
## AIC: 1764.01
vip(DryBean_TDA_KDE_5.50.5_n4_LrFit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n4_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n4_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n4_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n4_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n4_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        1      0    0      958     3    10   92
##   HOROZ          26      0  264       11   571     0  138
##   SEKER         366    156  224       38     0   586   36
##   SIRA            3      0    1       56     4    12  524
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6468          
##                  95% CI : (0.6319, 0.6615)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.5678          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9012
## Specificity                  1.00000       1.00000      1.0000          0.9649
## Pos Pred Value                   NaN           NaN         NaN          0.9004
## Neg Pred Value               0.90294       0.96176      0.8801          0.9652
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2348
## Detection Prevalence         0.00000       0.00000      0.0000          0.2608
## Balanced Accuracy            0.50000       0.50000      0.5000          0.9330
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9879       0.9638      0.6633
## Specificity                0.8746       0.7638      0.9769
## Pos Pred Value             0.5653       0.4168      0.8733
## Neg Pred Value             0.9977       0.9918      0.9236
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1400       0.1436      0.1284
## Detection Prevalence       0.2475       0.3446      0.1471
## Balanced Accuracy          0.9313       0.8638      0.8201
nb_tda_kde_5.50.5_n4_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        1      0    0      958     3    10   92
##   HOROZ          26      0  264       11   571     0  138
##   SEKER         366    156  224       38     0   586   36
##   SIRA            3      0    1       56     4    12  524
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6468          
##                  95% CI : (0.6319, 0.6615)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.5678          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.9012
## Specificity                  1.00000       1.00000      1.0000          0.9649
## Pos Pred Value                   NaN           NaN         NaN          0.9004
## Neg Pred Value               0.90294       0.96176      0.8801          0.9652
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.2348
## Detection Prevalence         0.00000       0.00000      0.0000          0.2608
## Balanced Accuracy            0.50000       0.50000      0.5000          0.9330
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9879       0.9638      0.6633
## Specificity                0.8746       0.7638      0.9769
## Pos Pred Value             0.5653       0.4168      0.8733
## Neg Pred Value             0.9977       0.9918      0.9236
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1400       0.1436      0.1284
## Detection Prevalence       0.2475       0.3446      0.1471
## Balanced Accuracy          0.9313       0.8638      0.8201
nb_tda_kde_5.50.5_n4_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.6468137      0.5677871      0.6319242      0.6614927      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
nb_tda_kde_5.50.5_n4_db_lr_cf0_ov_acc<-nb_tda_kde_5.50.5_n4_db_lr_cf0$overall[1]
nb_tda_kde_5.50.5_n4_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.0000000   1.0000000            NaN      0.9029412        NA
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647        NA
## Class: CALI       0.0000000   1.0000000            NaN      0.8801471        NA
## Class: DERMASON   0.9012230   0.9648658      0.9003759      0.9651857 0.9003759
## Class: HOROZ      0.9878893   0.8746431      0.5653465      0.9977199 0.5653465
## Class: SEKER      0.9638158   0.7638249      0.4167852      0.9917726 0.4167852
## Class: SIRA       0.6632911   0.9768997      0.8733333      0.9235632 0.8733333
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.0000000        NA 0.09705882      0.0000000
## Class: BOMBAY   0.0000000        NA 0.03823529      0.0000000
## Class: CALI     0.0000000        NA 0.11985294      0.0000000
## Class: DERMASON 0.9012230 0.9007992 0.26053922      0.2348039
## Class: HOROZ    0.9878893 0.7191436 0.14166667      0.1399510
## Class: SEKER    0.9638158 0.5819265 0.14901961      0.1436275
## Class: SIRA     0.6632911 0.7539568 0.19362745      0.1284314
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA            0.0000000         0.5000000
## Class: BOMBAY              0.0000000         0.5000000
## Class: CALI                0.0000000         0.5000000
## Class: DERMASON            0.2607843         0.9330444
## Class: HOROZ               0.2475490         0.9312662
## Class: SEKER               0.3446078         0.8638203
## Class: SIRA                0.1470588         0.8200954
nb_tda_kde_5.50.5_n4_db_lr_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n4_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_lr_n4_3_fold<-(db_lr_fit_re - nb_tda_kde_5.50.5_n4_lr_fit_re)
diff_drybean_tda_kde_5.50.5_lr_n4_3_fold
##     Accuracy
## 1 0.09047206
## 2 0.09843173
## 3 0.11197566
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n4_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n4_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n4_3_fold_odds.left<-bst_tda_kde_5.50.5_lr.n4_3_fold$probLeft/bst_tda_kde_5.50.5_lr.n4_3_fold$probRight
bst_tda_kde_5.50.5_lr.n4_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n4_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n4_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.009166667
## 
## $winRight
## [1] 0.9908333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n4_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n4_3_fold
## $left
## [1] 0.002145385
## 
## $rope
## [1] 0.001045579
## 
## $right
## [1] 0.996809
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_lr_n4_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr_n4_3_fold))
#bf_tda_kde_5.50.5_lr.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_lr_n4_3_fold)
## t = 15.978, df = 2, p-value = 0.003894
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.07328566 0.12730063
## sample estimates:
## mean of x 
## 0.1002931
### Test set diff
diff_drybean_tda_kde_5.50.5_lr.n4_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n4_db_lr_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_lr.n4_test
##  Accuracy 
## 0.2801471
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n4_test),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n4_test_odds.left<-bst_tda_kde_5.50.5_lr.n4_test$probLeft/bst_tda_kde_5.50.5_lr.n4_test$probRight
bst_tda_kde_5.50.5_lr.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n4_test),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1603667
## 
## $winRight
## [1] 0.8396333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n4_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_lr.n4_test)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr.n4_test)) #bf_tda_pca_5.50.5_lr.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n4_test))

##Node5

DryBean_TDA_KDE_5.50.5_n5_LrFit0 <- train(as.factor(Class) ~ ., 
                 data = tda.m_kde_dry_bean_dataset_5.50.5.n5.vec, 
                      family = 'binomial',
                            method = 'multinom', 
                      trControl = fitControl,
                            metric='Accuracy')
## # weights:  72 (51 variable)
## initial  value 913.567984 
## iter  10 value 473.572062
## iter  20 value 360.487467
## iter  30 value 338.976113
## iter  40 value 336.934926
## iter  50 value 333.715995
## iter  60 value 332.450956
## iter  70 value 331.726934
## iter  80 value 327.180446
## iter  90 value 325.558314
## iter 100 value 325.266000
## final  value 325.266000 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 913.567984 
## iter  10 value 473.575327
## iter  20 value 365.671145
## iter  30 value 352.893390
## iter  40 value 352.798874
## final  value 352.797176 
## converged
## # weights:  72 (51 variable)
## initial  value 913.567984 
## iter  10 value 473.572066
## iter  20 value 360.493681
## iter  30 value 339.024071
## iter  40 value 337.096165
## iter  50 value 334.492069
## iter  60 value 333.668045
## iter  70 value 333.334205
## iter  80 value 332.496906
## iter  90 value 332.426894
## iter 100 value 332.400301
## final  value 332.400301 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 914.954278 
## iter  10 value 477.859914
## iter  20 value 376.091907
## iter  30 value 349.283488
## iter  40 value 348.892466
## iter  50 value 347.999198
## iter  60 value 347.757715
## iter  70 value 347.382708
## iter  80 value 347.025186
## iter  90 value 345.019888
## iter 100 value 343.968336
## final  value 343.968336 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 914.954278 
## iter  10 value 477.863710
## iter  20 value 375.813524
## iter  30 value 359.838955
## iter  40 value 359.335738
## final  value 359.335369 
## converged
## # weights:  72 (51 variable)
## initial  value 914.954278 
## iter  10 value 477.859917
## iter  20 value 376.101644
## iter  30 value 349.320089
## iter  40 value 348.948253
## iter  50 value 348.240602
## iter  60 value 348.089343
## iter  70 value 347.899391
## iter  80 value 347.790865
## iter  90 value 347.611423
## iter 100 value 347.588855
## final  value 347.588855 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 913.567984 
## iter  10 value 478.335654
## iter  20 value 371.788064
## iter  30 value 363.890815
## iter  40 value 362.233731
## iter  50 value 360.031004
## iter  60 value 359.509521
## iter  70 value 356.949748
## iter  80 value 354.182153
## iter  90 value 353.332066
## iter 100 value 352.492464
## final  value 352.492464 
## stopped after 100 iterations
## # weights:  72 (51 variable)
## initial  value 913.567984 
## iter  10 value 478.340207
## iter  20 value 377.227567
## iter  30 value 372.436927
## iter  40 value 372.345430
## final  value 372.343912 
## converged
## # weights:  72 (51 variable)
## initial  value 913.567984 
## iter  10 value 478.335659
## iter  20 value 371.798023
## iter  30 value 363.914342
## iter  40 value 362.404538
## iter  50 value 360.811794
## iter  60 value 360.526530
## iter  70 value 360.041338
## iter  80 value 359.999631
## iter  90 value 359.948485
## final  value 359.925901 
## converged
## # weights:  72 (51 variable)
## initial  value 1371.045123 
## iter  10 value 809.052677
## iter  20 value 618.093157
## iter  30 value 545.533603
## iter  40 value 543.074284
## final  value 542.911194 
## converged
DryBean_TDA_KDE_5.50.5_n5_LrFit0
## Penalized Multinomial Regression 
## 
## 989 samples
##  16 predictor
##   4 classes: 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 659, 660, 659 
## Resampling results across tuning parameters:
## 
##   decay  Accuracy   Kappa    
##   0e+00  0.7482331  0.5462197
##   1e-04  0.7492432  0.5488360
##   1e-01  0.7522796  0.5477844
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was decay = 0.1.
DryBean_TDA_KDE_5.50.5_n5_LrFit0$resample
##    Accuracy     Kappa Resample
## 1 0.7545455 0.5523963    Fold1
## 2 0.7454545 0.5384923    Fold3
## 3 0.7568389 0.5524647    Fold2
nb_tda_kde_5.50.5_n5_lr_fit_re<-DryBean_TDA_KDE_5.50.5_n5_LrFit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n5_LrFit0)
## Call:
## nnet::multinom(formula = .outcome ~ ., data = dat, decay = param$decay, 
##     family = "binomial")
## 
## Coefficients:
##        (Intercept)         Area    Perimeter MajorAxisLength MinorAxisLength
## HOROZ  0.007046354 -0.007017273 -0.009861693     -0.07326776      -0.7085975
## SEKER  0.039912415  0.019275103  0.210052507     -2.65315936      -2.8076537
## SIRA  -0.058476682  0.002591046  0.105103512      0.69262706       1.0713977
##       AspectRation Eccentricity   ConvexArea EquivDiameter    Extent
## HOROZ   0.01384804   0.01064355  0.005671292     0.8895502 0.1086813
## SEKER   0.03869226  -0.03413745 -0.017839900     4.6033418 0.3133238
## SIRA   -0.39192565   0.30903918 -0.001628880    -2.2510964 4.6101021
##           Solidity   roundness  Compactness  ShapeFactor1  ShapeFactor2
## HOROZ  0.007371226  0.01399459  0.008317498  3.231153e-05  2.904234e-05
## SEKER  0.039671461  0.08392669  0.080927850 -3.331845e-05  6.506517e-04
## SIRA  -0.046568170 -0.16807132 -0.116932534 -1.159902e-03 -1.246201e-03
##       ShapeFactor3 ShapeFactor4
## HOROZ  0.008120907   0.01861115
## SEKER  0.115165415   0.11418006
## SIRA  -0.215281243  -0.17422578
## 
## Std. Errors:
##        (Intercept)        Area   Perimeter MajorAxisLength MinorAxisLength
## HOROZ 7.464852e-07 0.006744161 0.001017042    0.0001903871    1.168177e-06
## SEKER 2.380020e-05 0.002211257 0.010005601    0.0030792136    2.275544e-03
## SIRA  8.937963e-06 0.001099197 0.004150226    0.0023576757    6.086535e-04
##       AspectRation Eccentricity  ConvexArea EquivDiameter       Extent
## HOROZ 2.242900e-06 9.582827e-07 0.006634756  6.042939e-05 9.410022e-07
## SEKER 3.240715e-05 1.602362e-05 0.002310454  2.642834e-03 1.646313e-05
## SIRA  2.615937e-05 1.227216e-05 0.001129689  9.824652e-04 5.717465e-06
##           Solidity    roundness  Compactness ShapeFactor1 ShapeFactor2
## HOROZ 3.736457e-07 1.343551e-06 2.540216e-07 1.210208e-08 1.541372e-10
## SEKER 2.361202e-05 1.868319e-05 2.040759e-05 2.393448e-07 7.962231e-08
## SIRA  8.860390e-06 5.857470e-06 5.060119e-06 1.235992e-07 2.113535e-08
##       ShapeFactor3 ShapeFactor4
## HOROZ 7.372696e-08 5.898251e-07
## SEKER 1.751545e-05 2.375758e-05
## SIRA  4.646547e-06 8.917319e-06
## 
## Residual Deviance: 1085.822 
## AIC: 1187.822
vip(DryBean_TDA_KDE_5.50.5_n5_LrFit0,50) + ggtitle("DryBean_TDA_KDE_5.50.5_n5_Lr1Fit TDA-Assited LR")

# Predict outcome using DryBean_TDA_KDE_5.50.5_n5_LrFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n5_LrFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n5_db_lr_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length

## Warning in levels(reference) != levels(data): Levels are not in the same order
## for reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n5_db_lr_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0      751     1    10   80
##   HOROZ           0      0    0      213    10     0    1
##   SEKER          68     68    1       16     0   576    6
##   SIRA          328     88  488       83   567    22  703
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5             
##                  95% CI : (0.4845, 0.5155)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.3777          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.7065
## Specificity                  1.00000       1.00000      1.0000          0.9698
## Pos Pred Value                   NaN           NaN         NaN          0.8919
## Neg Pred Value               0.90294       0.96176      0.8801          0.9036
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.1841
## Detection Prevalence         0.00000       0.00000      0.0000          0.2064
## Balanced Accuracy            0.50000       0.50000      0.5000          0.8382
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity              0.017301       0.9474      0.8899
## Specificity              0.938892       0.9542      0.5210
## Pos Pred Value           0.044643       0.7837      0.3085
## Neg Pred Value           0.852697       0.9904      0.9517
## Prevalence               0.141667       0.1490      0.1936
## Detection Rate           0.002451       0.1412      0.1723
## Detection Prevalence     0.054902       0.1801      0.5586
## Balanced Accuracy        0.478097       0.9508      0.7054
nb_tda_kde_5.50.5_n5_db_lr_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA        0      0    0        0     0     0    0
##   BOMBAY          0      0    0        0     0     0    0
##   CALI            0      0    0        0     0     0    0
##   DERMASON        0      0    0      751     1    10   80
##   HOROZ           0      0    0      213    10     0    1
##   SEKER          68     68    1       16     0   576    6
##   SIRA          328     88  488       83   567    22  703
## 
## Overall Statistics
##                                           
##                Accuracy : 0.5             
##                  95% CI : (0.4845, 0.5155)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.3777          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.00000       0.00000      0.0000          0.7065
## Specificity                  1.00000       1.00000      1.0000          0.9698
## Pos Pred Value                   NaN           NaN         NaN          0.8919
## Neg Pred Value               0.90294       0.96176      0.8801          0.9036
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.00000       0.00000      0.0000          0.1841
## Detection Prevalence         0.00000       0.00000      0.0000          0.2064
## Balanced Accuracy            0.50000       0.50000      0.5000          0.8382
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity              0.017301       0.9474      0.8899
## Specificity              0.938892       0.9542      0.5210
## Pos Pred Value           0.044643       0.7837      0.3085
## Neg Pred Value           0.852697       0.9904      0.9517
## Prevalence               0.141667       0.1490      0.1936
## Detection Rate           0.002451       0.1412      0.1723
## Detection Prevalence     0.054902       0.1801      0.5586
## Balanced Accuracy        0.478097       0.9508      0.7054
nb_tda_kde_5.50.5_n5_db_lr_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   5.000000e-01   3.776857e-01   4.845399e-01   5.154601e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##  2.853660e-233            NaN
nb_tda_kde_5.50.5_n5_db_lr_cf0_ov_acc<-nb_tda_kde_5.50.5_n5_db_lr_cf0$overall[1]
nb_tda_kde_5.50.5_n5_db_lr_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value
## Class: BARBUNYA  0.00000000   1.0000000            NaN      0.9029412
## Class: BOMBAY    0.00000000   1.0000000            NaN      0.9617647
## Class: CALI      0.00000000   1.0000000            NaN      0.8801471
## Class: DERMASON  0.70649106   0.9698376     0.89192399      0.9036442
## Class: HOROZ     0.01730104   0.9388921     0.04464286      0.8526971
## Class: SEKER     0.94736842   0.9542051     0.78367347      0.9904335
## Class: SIRA      0.88987342   0.5209726     0.30846863      0.9516935
##                  Precision     Recall         F1 Prevalence Detection Rate
## Class: BARBUNYA         NA 0.00000000         NA 0.09705882     0.00000000
## Class: BOMBAY           NA 0.00000000         NA 0.03823529     0.00000000
## Class: CALI             NA 0.00000000         NA 0.11985294     0.00000000
## Class: DERMASON 0.89192399 0.70649106 0.78845144 0.26053922     0.18406863
## Class: HOROZ    0.04464286 0.01730104 0.02493766 0.14166667     0.00245098
## Class: SEKER    0.78367347 0.94736842 0.85778109 0.14901961     0.14117647
## Class: SIRA     0.30846863 0.88987342 0.45812968 0.19362745     0.17230392
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.00000000         0.5000000
## Class: BOMBAY             0.00000000         0.5000000
## Class: CALI               0.00000000         0.5000000
## Class: DERMASON           0.20637255         0.8381643
## Class: HOROZ              0.05490196         0.4780965
## Class: SEKER              0.18014706         0.9507867
## Class: SIRA               0.55857843         0.7054230
nb_tda_kde_5.50.5_n5_db_lr_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n5_db_lr_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_lr_n5_3_fold<-(db_lr_fit_re - nb_tda_kde_5.50.5_n5_lr_fit_re)
diff_drybean_tda_kde_5.50.5_lr_n5_3_fold
##    Accuracy
## 1 0.1818525
## 2 0.1749356
## 3 0.1669886
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n5_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n5_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n5_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n5_3_fold_odds.left<-bst_tda_kde_5.50.5_lr.n5_3_fold$probLeft/bst_tda_kde_5.50.5_lr.n5_3_fold$probRight
bst_tda_kde_5.50.5_lr.n5_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n5_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n5_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n5_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.009966667
## 
## $winRight
## [1] 0.9900333
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n5_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n5_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n5_3_fold
## $left
## [1] 0.0003604065
## 
## $rope
## [1] 9.278293e-05
## 
## $right
## [1] 0.9995468
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_lr_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr_n5_3_fold))
#bf_tda_kde_5.50.5_lr.n5_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr_n5_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_lr_n5_3_fold)
## t = 40.657, df = 2, p-value = 0.0006044
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.1561155 0.1930691
## sample estimates:
## mean of x 
## 0.1745923
### Test set diff
diff_drybean_tda_kde_5.50.5_lr.n5_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n5_db_lr_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_lr.n5_test
##  Accuracy 
## 0.4269608
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_lr.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n5_test),-0.01,0.01)
bst_tda_kde_5.50.5_lr.n5_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_lr.n5_test_odds.left<-bst_tda_kde_5.50.5_lr.n5_test$probLeft/bst_tda_kde_5.50.5_lr.n5_test$probRight
bst_tda_kde_5.50.5_lr.n5_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_lr.n5_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n5_test),-0.01,0.01)
bsr_tda_kde_5.50.5_lr.n5_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1575333
## 
## $winRight
## [1] 0.8424667
# Bayesian Correlated Test

bct_tda_kde_5.50.5_lr.n5_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n5_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_lr.n5_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_lr.n5_test)))

#BayesFactor
#bf_tda_kde_5.50.5_lr.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_lr.n5_test)) #bf_tda_pca_5.50.5_lr.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_lr.n5_test))


#naiveBayes 
dryBeanNbFit <- train(as.factor(Class) ~ ., data = Dry_Bean_DatasetTrain, 
                method = 'nb', 
                trControl = fitControl,
                metric='Accuracy')
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 20
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 826
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 838
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 882
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1203
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1322
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1339
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1812
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2931
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3092
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 20
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 96
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 826
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 838
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 868
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 882
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 915
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1203
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1261
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1265
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1270
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1315
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1812
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2017
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2036
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2049
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2226
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2931
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3092
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3107
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 5
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 27
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 857
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 890
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 896
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 897
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1045
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1309
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1905
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1983
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2807
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2865
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3053
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3099
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 27
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 570
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 857
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 890
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 896
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 897
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1045
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1147
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1546
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1872
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1905
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2497
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2807
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3053
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3099
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 7
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 72
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 881
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 910
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2347
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 72
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 519
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 840
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 881
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 910
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 914
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 915
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1303
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1601
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2087
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2215
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2345
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2618
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2838
dryBeanNbFit
## Naive Bayes 
## 
## 9531 samples
##   16 predictor
##    7 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 6353, 6354, 6355 
## Resampling results across tuning parameters:
## 
##   usekernel  Accuracy   Kappa    
##   FALSE      0.8986477  0.8775870
##    TRUE      0.9019001  0.8814099
## 
## Tuning parameter 'fL' was held constant at a value of 0
## Tuning
##  parameter 'adjust' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were fL = 0, usekernel = TRUE and adjust
##  = 1.
dryBeanNbFit$resample
##    Accuracy     Kappa Resample
## 1 0.8967904 0.8752535    Fold1
## 2 0.9017941 0.8812123    Fold2
## 3 0.9071159 0.8877637    Fold3
db_nb_fit_re<-dryBeanNbFit$resample[1]

summary(dryBeanNbFit)
##             Length Class      Mode     
## apriori      7     table      numeric  
## tables      16     -none-     list     
## levels       7     -none-     character
## call         6     -none-     call     
## x           16     data.frame list     
## usekernel    1     -none-     logical  
## varnames    16     -none-     character
## xNames      16     -none-     character
## problemType  1     -none-     character
## tuneValue    3     data.frame list     
## obsLevels    7     -none-     character
## param        0     -none-     list
#varImp (dryBeanNbFit)



# Predict outcome using model from training data based on testing data
predictions <- predict(dryBeanNbFit, newdata= Dry_Bean_DatasetTest)
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 5
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 36
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 93
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 125
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 595
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 603
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 604
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 605
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 606
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 641
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 800
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 961
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 993
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 995
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 996
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 998
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 999
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1000
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1001
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1002
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1013
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1552
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1642
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1643
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2219
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2220
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2223
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2226
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2938
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2973
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2978
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3008
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3019
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3020
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3021
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3022
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3024
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3025
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3034
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3035
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3037
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3042
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3043
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3047
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3056
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3060
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3061
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3065
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3089
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3102
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3114
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3898
# Create confusion matrix to assess model fit/performance on test data
db_nb_cf<-confusionMatrix(data=predictions, as.factor(Dry_Bean_DatasetTest$Class))
db_nb_cf
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      319      0   42        0     2     5    6
##   BOMBAY          1    156    0        0     0     0    0
##   CALI           56      0  433        0    11     0    1
##   DERMASON        0      0    0      940     6     6   76
##   HOROZ           3      0    9        1   545     0   19
##   SEKER           0      0    0       30     0   575    7
##   SIRA           17      0    5       92    14    22  681
## 
## Overall Statistics
##                                           
##                Accuracy : 0.8944          
##                  95% CI : (0.8845, 0.9036)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.8723          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.80556       1.00000      0.8855          0.8843
## Specificity                  0.98507       0.99975      0.9811          0.9708
## Pos Pred Value               0.85294       0.99363      0.8643          0.9144
## Neg Pred Value               0.97922       1.00000      0.9844          0.9597
## Prevalence                   0.09706       0.03824      0.1199          0.2605
## Detection Rate               0.07819       0.03824      0.1061          0.2304
## Detection Prevalence         0.09167       0.03848      0.1228          0.2520
## Balanced Accuracy            0.89531       0.99987      0.9333          0.9276
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9429       0.9457      0.8620
## Specificity                0.9909       0.9893      0.9544
## Pos Pred Value             0.9445       0.9395      0.8195
## Neg Pred Value             0.9906       0.9905      0.9665
## Prevalence                 0.1417       0.1490      0.1936
## Detection Rate             0.1336       0.1409      0.1669
## Detection Prevalence       0.1414       0.1500      0.2037
## Balanced Accuracy          0.9669       0.9675      0.9082
db_nb_cf$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.8943627      0.8723330      0.8845246      0.9036320      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_nb_cf_ov_acc<-db_nb_cf$overall[1]
db_nb_cf$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.8055556   0.9850706      0.8529412      0.9792229 0.8529412
## Class: BOMBAY     1.0000000   0.9997452      0.9936306      1.0000000 0.9936306
## Class: CALI       0.8854806   0.9810638      0.8642715      0.9843532 0.8642715
## Class: DERMASON   0.8842897   0.9708320      0.9143969      0.9596986 0.9143969
## Class: HOROZ      0.9429066   0.9908624      0.9445407      0.9905795 0.9445407
## Class: SEKER      0.9457237   0.9893433      0.9395425      0.9904844 0.9395425
## Class: SIRA       0.8620253   0.9544073      0.8194946      0.9664512 0.8194946
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.8055556 0.8285714 0.09705882     0.07818627
## Class: BOMBAY   1.0000000 0.9968051 0.03823529     0.03823529
## Class: CALI     0.8854806 0.8747475 0.11985294     0.10612745
## Class: DERMASON 0.8842897 0.8990913 0.26053922     0.23039216
## Class: HOROZ    0.9429066 0.9437229 0.14166667     0.13357843
## Class: SEKER    0.9457237 0.9426230 0.14901961     0.14093137
## Class: SIRA     0.8620253 0.8402221 0.19362745     0.16691176
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.09166667         0.8953131
## Class: BOMBAY             0.03848039         0.9998726
## Class: CALI               0.12279412         0.9332722
## Class: DERMASON           0.25196078         0.9275608
## Class: HOROZ              0.14142157         0.9668845
## Class: SEKER              0.15000000         0.9675335
## Class: SIRA               0.20367647         0.9082163
db_nb_cf_pre_rec_f1<-db_nb_cf$byClass[5:7]


##With TDA PCA filter 5 intervals, 50% overlap, 5 bins 
##Node1

DryBean_TDA_PC_5.50.5_n1_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n1.vec, 
                method = 'nb', 
                trControl = fitControl,
                metric='Accuracy')
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 13
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 38
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 67
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 621
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 924
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1077
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1946
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2117
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2334
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 38
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 67
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 924
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 947
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1071
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1077
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1080
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1508
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1533
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1581
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1614
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1932
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1941
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2117
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 7
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 32
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 130
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 562
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 879
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1894
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2031
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2085
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 7
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 32
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 130
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 879
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1894
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2031
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2072
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2085
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2509
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 9
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 38
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 49
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 135
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1808
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2083
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 9
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 38
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 49
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 135
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 969
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1064
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2083
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2458
DryBean_TDA_PC_5.50.5_n1_NbFit0
## Naive Bayes 
## 
## 7839 samples
##   16 predictor
##    6 classes: 'BARBUNYA', 'CALI', 'DERMASON', 'HOROZ', 'SEKER', 'SIRA' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 5227, 5225, 5226 
## Resampling results across tuning parameters:
## 
##   usekernel  Accuracy   Kappa   
##   FALSE      0.8534248  0.783064
##    TRUE      0.8585274  0.789569
## 
## Tuning parameter 'fL' was held constant at a value of 0
## Tuning
##  parameter 'adjust' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were fL = 0, usekernel = TRUE and adjust
##  = 1.
DryBean_TDA_PC_5.50.5_n1_NbFit0$resample
##    Accuracy     Kappa Resample
## 1 0.8526034 0.7812269    Fold1
## 2 0.8565417 0.7870815    Fold2
## 3 0.8664370 0.8003985    Fold3
db_tda_pc_5.50.5_n1_nb_fit_re<-DryBean_TDA_PC_5.50.5_n1_NbFit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n1_NbFit0)
##             Length Class      Mode     
## apriori      6     table      numeric  
## tables      16     -none-     list     
## levels       6     -none-     character
## call         6     -none-     call     
## x           16     data.frame list     
## usekernel    1     -none-     logical  
## varnames    16     -none-     character
## xNames      16     -none-     character
## problemType  1     -none-     character
## tuneValue    3     data.frame list     
## obsLevels    6     -none-     character
## param        0     -none-     list
# Predict outcome using DryBean_TDA_PC_5.50.5_n1_NbFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n1_NbFit0, newdata= Dry_Bean_DatasetTest)
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 5
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 36
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 93
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 125
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 634
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 635
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 706
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 744
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 795
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 796
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 798
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 800
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 802
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 803
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 805
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 807
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 808
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 809
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 812
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 814
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 816
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 818
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 819
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 823
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 825
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 826
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 827
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 828
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 830
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 832
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 833
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 834
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 838
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 839
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 840
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 841
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 842
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 843
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 844
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 845
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 846
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 847
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 848
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 849
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 851
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 852
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 854
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 855
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 856
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 857
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 859
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 860
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 861
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 863
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 864
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 865
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 866
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 868
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 869
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 871
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 872
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 873
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 875
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 879
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 881
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 882
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 884
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 886
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 888
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 890
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 892
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 893
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 894
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 896
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 897
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 905
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 910
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 912
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 913
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 914
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 915
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 916
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 918
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 920
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 922
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 923
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 924
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 926
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 927
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 929
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 930
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 931
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 932
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 933
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 934
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 935
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 936
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 938
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 939
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 941
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 942
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 943
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 944
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 945
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 946
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 947
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 948
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 949
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 950
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 951
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 952
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 955
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 957
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 959
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 960
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 961
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 962
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 963
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 964
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 965
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 967
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 968
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 969
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 972
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 973
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 974
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 975
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 976
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 977
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 978
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 979
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 980
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 981
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 982
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 983
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 987
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 988
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 989
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 990
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 991
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 993
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 994
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 995
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 996
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 997
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 998
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 999
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1000
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1001
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1002
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1008
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1009
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1013
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1014
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1015
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1017
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1019
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1020
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1021
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1022
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1024
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1025
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1027
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1029
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1030
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1031
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1032
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1033
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1034
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1035
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1036
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1037
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1038
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1039
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1040
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1041
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1042
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1043
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1045
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1046
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1047
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1048
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1049
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1053
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1055
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1056
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1057
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1058
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1059
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1060
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1061
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1062
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1063
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1064
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1065
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1066
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1067
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1068
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1069
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1070
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1071
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1072
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1073
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1075
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1076
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1077
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1079
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1080
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1081
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1082
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1083
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1084
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1085
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1086
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1087
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1088
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1089
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1090
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1091
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1092
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1093
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1094
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1095
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1096
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1097
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1098
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1099
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1100
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1102
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1106
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1107
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1108
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1112
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1113
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1114
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1115
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1116
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1117
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1118
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1119
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1120
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1122
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1123
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1124
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1125
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1126
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1128
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1129
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1130
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1131
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1133
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1135
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1136
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1138
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1140
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1141
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1144
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1147
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1148
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1162
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1164
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1166
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1168
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1171
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1172
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1173
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1175
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1177
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1178
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1179
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1181
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1182
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1184
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1185
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1188
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1190
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1199
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1201
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1202
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1203
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1204
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1205
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1210
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1213
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1215
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1216
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1219
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1220
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1221
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1222
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1223
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1225
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1226
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1229
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1230
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1232
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1233
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1238
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1242
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1244
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1245
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1246
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1247
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1248
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1251
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1252
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1259
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1260
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1261
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1262
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1263
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1265
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1266
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1270
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1303
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1308
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1309
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1310
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1311
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1312
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1315
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1316
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1318
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1319
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1321
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1322
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1324
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1325
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1326
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1327
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1329
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1331
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1332
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1334
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1336
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1337
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1338
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1339
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1341
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1344
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1345
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1347
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1349
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1362
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1363
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1364
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1365
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1366
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1374
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1375
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1377
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1379
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1385
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1387
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1390
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1392
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1394
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1400
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1402
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1406
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1411
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1451
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1475
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1481
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1497
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1498
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1504
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1508
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1509
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1514
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1515
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1519
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1520
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1521
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1523
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1524
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1531
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1533
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1535
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1536
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1543
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1544
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1546
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1547
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1549
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1551
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1552
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1553
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1555
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1558
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1559
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1560
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1561
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1562
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1563
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1566
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1569
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1570
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1574
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1576
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1580
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1581
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1582
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1584
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1585
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1587
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1589
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1590
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1591
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1592
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1593
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1594
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1595
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1598
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1599
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1601
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1602
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1603
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1604
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1605
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1606
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1607
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1612
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1613
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1614
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1616
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1618
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1621
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1627
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1633
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1634
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1635
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1638
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1640
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1641
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1642
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1643
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1644
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1706
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1744
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1795
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1796
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1798
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1800
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1802
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1803
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1805
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1807
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1808
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1809
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1812
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1814
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1816
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1818
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1819
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1823
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1825
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1826
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1827
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1828
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1830
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1832
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1833
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1834
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1838
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1839
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1840
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1841
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1842
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1843
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1844
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1845
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1846
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1847
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1848
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1849
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1851
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1852
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1854
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1855
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1856
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1857
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1859
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1860
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1861
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1863
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1864
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1865
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1866
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1868
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1869
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1871
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1872
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1873
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1875
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1879
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1881
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1882
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1884
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1886
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1888
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1890
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1892
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1893
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1894
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1896
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1897
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1905
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1910
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1912
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1913
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1914
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1915
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1916
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1918
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1920
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1922
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1923
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1924
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1926
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1927
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1929
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1930
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1931
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1932
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1933
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1934
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1935
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1936
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1938
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1939
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1941
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1942
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1943
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1944
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1945
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1946
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1947
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1948
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1949
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1950
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1951
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1952
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1955
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1957
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1959
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1960
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1961
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1962
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1963
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1964
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1965
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1967
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1968
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1969
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1972
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1973
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1974
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1975
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1976
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1977
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1978
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1979
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1980
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1981
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1982
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1983
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1987
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1988
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1989
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1990
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1991
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1993
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1994
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1995
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1996
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1997
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1998
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1999
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2000
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2001
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2002
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2008
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2009
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2013
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2014
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2015
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2017
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2019
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2020
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2021
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2022
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2024
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2025
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2027
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2029
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2030
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2031
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2032
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2033
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2034
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2035
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2036
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2037
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2038
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2039
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2040
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2041
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2042
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2043
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2045
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2046
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2047
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2048
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2049
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2053
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2055
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2056
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2057
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2058
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2059
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2060
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2061
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2062
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2063
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2064
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2065
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2066
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2067
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2068
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2069
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2070
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2071
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2072
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2073
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2075
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2076
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2077
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2079
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2080
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2081
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2082
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2083
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2084
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2085
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2086
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2087
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2088
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2089
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2090
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2091
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2092
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2093
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2094
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2095
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2096
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2097
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2098
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2099
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2100
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2102
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2106
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2107
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2108
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2112
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2113
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2114
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2115
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2116
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2117
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2118
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2119
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2120
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2122
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2123
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2124
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2125
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2126
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2128
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2129
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2130
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2131
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2133
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2135
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2136
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2138
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2140
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2141
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2144
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2147
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2148
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2161
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2162
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2163
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2164
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2166
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2168
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2171
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2172
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2173
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2175
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2177
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2178
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2179
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2181
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2182
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2184
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2185
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2188
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2190
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2199
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2201
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2202
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2203
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2204
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2205
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2210
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2213
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2215
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2216
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2219
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2220
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2221
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2222
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2223
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2225
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2226
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2543
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2819
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2823
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2938
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2941
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2942
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2948
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2949
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2962
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2963
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2965
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2969
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2973
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2978
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2979
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2983
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2987
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2989
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2990
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2991
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2993
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2994
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2995
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2996
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2998
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2999
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3000
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3001
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3002
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3008
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3009
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3013
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3014
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3015
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3017
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3019
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3020
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3021
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3022
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3024
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3025
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3034
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3035
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3037
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3042
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3043
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3047
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3060
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3061
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3065
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3089
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3102
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3628
# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n1_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n1_db_nb_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      294      0  183        0   253    17   15
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           36     78   29        0   209     9    9
##   DERMASON        0      0    0      873     1     6   59
##   HOROZ          60     78  274       72   108     0   39
##   SEKER           0      0    0       30     0   562    6
##   SIRA            6      0    3       88     7    14  662
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6196          
##                  95% CI : (0.6045, 0.6345)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.5418          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.74242       0.00000    0.059305          0.8213
## Specificity                  0.87296       1.00000    0.905040          0.9781
## Pos Pred Value               0.38583           NaN    0.078378          0.9297
## Neg Pred Value               0.96926       0.96176    0.876011          0.9395
## Prevalence                   0.09706       0.03824    0.119853          0.2605
## Detection Rate               0.07206       0.00000    0.007108          0.2140
## Detection Prevalence         0.18676       0.00000    0.090686          0.2301
## Balanced Accuracy            0.80769       0.50000    0.482173          0.8997
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity               0.18685       0.9243      0.8380
## Specificity               0.85066       0.9896      0.9641
## Pos Pred Value            0.17116       0.9398      0.8487
## Neg Pred Value            0.86373       0.9868      0.9612
## Prevalence                0.14167       0.1490      0.1936
## Detection Rate            0.02647       0.1377      0.1623
## Detection Prevalence      0.15466       0.1466      0.1912
## Balanced Accuracy         0.51875       0.9570      0.9011
db_tda_pc_5.50.5_n1_db_nb_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      294      0  183        0   253    17   15
##   BOMBAY          0      0    0        0     0     0    0
##   CALI           36     78   29        0   209     9    9
##   DERMASON        0      0    0      873     1     6   59
##   HOROZ          60     78  274       72   108     0   39
##   SEKER           0      0    0       30     0   562    6
##   SIRA            6      0    3       88     7    14  662
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6196          
##                  95% CI : (0.6045, 0.6345)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.5418          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.74242       0.00000    0.059305          0.8213
## Specificity                  0.87296       1.00000    0.905040          0.9781
## Pos Pred Value               0.38583           NaN    0.078378          0.9297
## Neg Pred Value               0.96926       0.96176    0.876011          0.9395
## Prevalence                   0.09706       0.03824    0.119853          0.2605
## Detection Rate               0.07206       0.00000    0.007108          0.2140
## Detection Prevalence         0.18676       0.00000    0.090686          0.2301
## Balanced Accuracy            0.80769       0.50000    0.482173          0.8997
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity               0.18685       0.9243      0.8380
## Specificity               0.85066       0.9896      0.9641
## Pos Pred Value            0.17116       0.9398      0.8487
## Neg Pred Value            0.86373       0.9868      0.9612
## Prevalence                0.14167       0.1490      0.1936
## Detection Rate            0.02647       0.1377      0.1623
## Detection Prevalence      0.15466       0.1466      0.1912
## Balanced Accuracy         0.51875       0.9570      0.9011
db_tda_pc_5.50.5_n1_db_nb_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##      0.6196078      0.5418470      0.6045071      0.6345371      0.2605392 
## AccuracyPValue  McnemarPValue 
##      0.0000000            NaN
db_tda_pc_5.50.5_n1_db_nb_cf0_ov_acc<-db_tda_pc_5.50.5_n1_db_nb_cf0$overall[1]
db_tda_pc_5.50.5_n1_db_nb_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value
## Class: BARBUNYA   0.7424242   0.8729642     0.38582677      0.9692586
## Class: BOMBAY     0.0000000   1.0000000            NaN      0.9617647
## Class: CALI       0.0593047   0.9050404     0.07837838      0.8760108
## Class: DERMASON   0.8212606   0.9781240     0.92971246      0.9395097
## Class: HOROZ      0.1868512   0.8506568     0.17115689      0.8637286
## Class: SEKER      0.9243421   0.9896313     0.93979933      0.9867892
## Class: SIRA       0.8379747   0.9641337     0.84871795      0.9612121
##                  Precision    Recall         F1 Prevalence Detection Rate
## Class: BARBUNYA 0.38582677 0.7424242 0.50777202 0.09705882    0.072058824
## Class: BOMBAY           NA 0.0000000         NA 0.03823529    0.000000000
## Class: CALI     0.07837838 0.0593047 0.06752037 0.11985294    0.007107843
## Class: DERMASON 0.92971246 0.8212606 0.87212787 0.26053922    0.213970588
## Class: HOROZ    0.17115689 0.1868512 0.17866005 0.14166667    0.026470588
## Class: SEKER    0.93979933 0.9243421 0.93200663 0.14901961    0.137745098
## Class: SIRA     0.84871795 0.8379747 0.84331210 0.19362745    0.162254902
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.18676471         0.8076942
## Class: BOMBAY             0.00000000         0.5000000
## Class: CALI               0.09068627         0.4821725
## Class: DERMASON           0.23014706         0.8996923
## Class: HOROZ              0.15465686         0.5187540
## Class: SEKER              0.14656863         0.9569867
## Class: SIRA               0.19117647         0.9010542
db_tda_pc_5.50.5_n1_db_nb_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n1_db_nb_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_nb_n1_3_fold<-(db_nb_fit_re - db_tda_pc_5.50.5_n1_nb_fit_re)
diff_drybean_tda_pca_5.50.5_nb_n1_3_fold
##     Accuracy
## 1 0.04418707
## 2 0.04525245
## 3 0.04067882
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nb.n1_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n1_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nb.n1_3_fold
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0.75
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nb.n1_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n1_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_nb.n1_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_nb.n1_3_fold_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nb.n1_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n1_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nb.n1_3_fold
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.009466667
## 
## $winRight
## [1] 0.9905333
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nb.n1_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n1_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nb.n1_3_fold
## $left
## [1] 0.0004461477
## 
## $rope
## [1] 0.0006926052
## 
## $right
## [1] 0.9988612
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_nb_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb_n1_3_fold))
#bf_tda_pca_5.50.5_nb.n1_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n1_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_nb_n1_3_fold)
## t = 31.392, df = 2, p-value = 0.001013
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.03742805 0.04931750
## sample estimates:
##  mean of x 
## 0.04337278
### Test set diff
diff_drybean_tda_pca_5.50.5_nb.n1_test<-(db_nb_cf_ov_acc - db_tda_pc_5.50.5_n1_db_nb_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_nb.n1_test
##  Accuracy 
## 0.2747549
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nb.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n1_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nb.n1_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nb.n1_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n1_test$probLeft/bst_dbf_db_tda_pca_5.50.5_nb.n1_test$probRight
bst_dbf_db_tda_pca_5.50.5_nb.n1_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nb.n1_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n1_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nb.n1_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1603333
## 
## $winRight
## [1] 0.8396667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nb.n1_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n1_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nb.n1_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nb.n1_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb.n1_test)) #bf_tda_pca_5.50.5_nb.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n1_test))

##Node2

#DryBean_TDA_PC_5.50.5_n2_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n2.vec, 
#                method = 'nb', 
#                trControl = fitControl,
#                metric='Accuracy')

#DryBean_TDA_PC_5.50.5_n2_NbFit0
#DryBean_TDA_PC_5.50.5_n2_NbFit0$resample
#db_tda_pc_5.50.5_n2_nb_fit_re<-DryBean_TDA_PC_5.50.5_n2_NbFit0$resample[1]

#summary(DryBean_TDA_PC_5.50.5_n2_NbFit0)

# Predict outcome using DryBean_TDA_PC_5.50.5_n2_NbFit0 from training data based on testing data
#pred0 <- predict(DryBean_TDA_PC_5.50.5_n2_NbFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
#db_tda_pc_5.50.5_n2_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
#db_tda_pc_5.50.5_n2_db_nb_cf0
#db_tda_pc_5.50.5_n2_db_nb_cf0 
#db_tda_pc_5.50.5_n2_db_nb_cf0$overall
#db_tda_pc_5.50.5_n2_db_nb_cf0_ov_acc<-db_tda_pc_5.50.5_n2_db_nb_cf0$overall[1]
#db_tda_pc_5.50.5_n2_db_nb_cf0$byClass
#db_tda_pc_5.50.5_n2_db_nb_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n2_db_nb_cf0$byClass[5:7]#

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

#diff_drybean_tda_pca_5.50.5_nb_n2_3_fold<-(db_nb_fit_re - db_tda_pc_5.50.5_n2_nb_fit_re)
#diff_drybean_tda_pca_5.50.5_nb_n2_3_fold

## Bayesian Tests 3-fold diff

# Bayesian Sign Test

#bst_dbf_db_tda_pca_5.50.5_nb.n2_3_fold<-#BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n2_3_fold),-0.01,0.01)
#bst_dbf_db_tda_pca_5.50.5_nb.n2_3_fold

# Odds Left Bayesian Sign Test 

#bst_dbf_db_tda_pca_5.50.5_nb.n2_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n2_3_fold$probLeft/#bst_dbf_db_tda_pca_5.50.5_nb.n2_3_fold$probRight
#bst_dbf_db_tda_pca_5.50.5_nb.n2_3_fold_odds.left

# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_nb.n2_3_fold<-#BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n2_3_fold),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_nb.n2_3_fold

# Bayesian Correlated Test

#bct_dbf_db_tda_pca_5.50.5_nb.n2_3_fold<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n2_3_fold),0.1,-0.01,0.01)
#bct_dbf_db_tda_pca_5.50.5_nb.n2_3_fold

# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nb_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb_n2_3_fold))
#bf_tda_pca_5.50.5_nb.n2_3_fold

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n2_3_fold))


### Test set diff
#diff_drybean_tda_pca_5.50.5_nb.n2_test<-(db_nb_cf_ov_acc - db_tda_pc_5.50.5_n2_db_nb_cf0_ov_acc)
#diff_drybean_tda_pca_5.50.5_nb.n2_test


## Bayesian Tests Test set diff

# Bayesian Sign Test

#bst_dbf_db_tda_pca_5.50.5_nb.n2_test<-#BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n2_test),-0.01,0.01)
#bst_dbf_db_tda_pca_5.50.5_nb.n2_test

# Odds Left Bayesian Sign Test 

#bst_dbf_db_tda_pca_5.50.5_nb.n2_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n2_test$probLeft/#bst_dbf_db_tda_pca_5.50.5_nb.n2_test$probRight
#bst_dbf_db_tda_pca_5.50.5_nb.n2_test_odds.left

# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_nb.n2_test<-#BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n2_test),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_nb.n2_test

# Bayesian Correlated Test

#bct_dbf_db_tda_pca_5.50.5_nb.n2_test<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n2_test),0.1,-0.01,0.01)
#bct_dbf_db_tda_pca_5.50.5_nb.n2_test

# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nb.n2_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb.n2_test)) #bf_tda_pca_5.50.5_nb.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n2_test))

##Node3

#DryBean_TDA_PC_5.50.5_n3_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n3.vec, 
#                method = 'nb', 
#                trControl = fitControl,
#                metric='Accuracy')

#DryBean_TDA_PC_5.50.5_n3_NbFit0
#DryBean_TDA_PC_5.50.5_n3_NbFit0$resample
#db_tda_pc_5.50.5_n3_nb_fit_re<-DryBean_TDA_PC_5.50.5_n3_NbFit0$resample[1]

#summary(DryBean_TDA_PC_5.50.5_n3_NbFit0)

# Predict outcome using DryBean_TDA_PC_5.50.5_n3_NbFit0 from training data based on testing data
#pred0 <- predict(DryBean_TDA_PC_5.50.5_n3_NbFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
#db_tda_pc_5.50.5_n3_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
#db_tda_pc_5.50.5_n3_db_nb_cf0
#db_tda_pc_5.50.5_n3_db_nb_cf0 
#db_tda_pc_5.50.5_n3_db_nb_cf0$overall
#db_tda_pc_5.50.5_n3_db_nb_cf0_ov_acc<-db_tda_pc_5.50.5_n3_db_nb_cf0$overall[1]
#db_tda_pc_5.50.5_n3_db_nb_cf0$byClass
#db_tda_pc_5.50.5_n3_db_nb_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n3_db_nb_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

#diff_drybean_tda_pca_5.50.5_nb_n3_3_fold<-(db_nb_fit_re - db_tda_pc_5.50.5_n3_nb_fit_re)
#diff_drybean_tda_pca_5.50.5_nb_n3_3_fold

## Bayesian Tests 3-fold diff

# Bayesian Sign Test

#bst_dbf_db_tda_pca_5.50.5_nb.n3_3_fold<-#BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n3_3_fold),-0.01,0.01)
#bst_dbf_db_tda_pca_5.50.5_nb.n3_3_fold

# Odds Left Bayesian Sign Test 

#bst_dbf_db_tda_pca_5.50.5_nb.n3_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n3_3_fold$probLeft/#bst_dbf_db_tda_pca_5.50.5_nb.n3_3_fold$probRight
#bst_dbf_db_tda_pca_5.50.5_nb.n3_3_fold_odds.left

# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_nb.n3_3_fold<-#BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n3_3_fold),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_nb.n3_3_fold

# Bayesian Correlated Test

#bct_dbf_db_tda_pca_5.50.5_nb.n3_3_fold<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n3_3_fold),0.1,-0.01,0.01)
#bct_dbf_db_tda_pca_5.50.5_nb.n3_3_fold

# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nb_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb_n3_3_fold))
#bf_tda_pca_5.50.5_nb.n3_3_fold

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n3_3_fold))


### Test set diff
#diff_drybean_tda_pca_5.50.5_nb.n3_test<-(db_nb_cf_ov_acc - db_tda_pc_5.50.5_n3_db_nb_cf0_ov_acc)
#diff_drybean_tda_pca_5.50.5_nb.n3_test


## Bayesian Tests Test set diff

# Bayesian Sign Test

#bst_dbf_db_tda_pca_5.50.5_nb.n3_test<-#BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n3_test),-0.01,0.01)
#bst_dbf_db_tda_pca_5.50.5_nb.n3_test

# Odds Left Bayesian Sign Test 

#bst_dbf_db_tda_pca_5.50.5_nb.n3_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n3_test$probLeft/#bst_dbf_db_tda_pca_5.50.5_nb.n3_test$probRight
#bst_dbf_db_tda_pca_5.50.5_nb.n3_test_odds.left

# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_nb.n2_test<-#BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n2_test),-0.01,0.01)
##bsr_dbf_db_tda_pca_5.50.5_nb.n2_test

# Bayesian Correlated Test

#bct_dbf_db_tda_pca_5.50.5_nb.n3_test<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n3_test),0.1,-0.01,0.01)
#bct_dbf_db_tda_pca_5.50.5_nb.n3_test

# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nb.n3_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb.n3_test)) #bf_tda_pca_5.50.5_nb.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n3_test))

##Node4

DryBean_TDA_PC_5.50.5_n4_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n4.vec, 
                method = 'nb', 
                trControl = fitControl,
                metric='Accuracy')
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 23
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 36
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 48
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 57
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 68
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 94
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 36
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 46
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 48
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 52
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 54
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 57
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 68
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 94
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 38
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 46
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 47
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 49
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 83
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 38
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 46
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 47
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 49
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 83
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 181
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 36
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 50
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 61
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 66
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 86
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 99
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 36
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 49
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 50
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 61
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 66
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 86
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 99
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 525
DryBean_TDA_PC_5.50.5_n4_NbFit0
## Naive Bayes 
## 
## 1590 samples
##   16 predictor
##    4 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'HOROZ' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1061, 1060, 1059 
## Resampling results across tuning parameters:
## 
##   usekernel  Accuracy   Kappa    
##   FALSE      0.9578615  0.9391147
##    TRUE      0.9566013  0.9371819
## 
## Tuning parameter 'fL' was held constant at a value of 0
## Tuning
##  parameter 'adjust' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were fL = 0, usekernel = FALSE and adjust
##  = 1.
DryBean_TDA_PC_5.50.5_n4_NbFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9565217 0.9373478    Fold1
## 2 0.9603774 0.9428498    Fold2
## 3 0.9566855 0.9371465    Fold3
db_tda_pc_5.50.5_n4_nb_fit_re<-DryBean_TDA_PC_5.50.5_n4_NbFit0$resample[1]

summary(DryBean_TDA_PC_5.50.5_n4_NbFit0)
##             Length Class      Mode     
## apriori      4     table      numeric  
## tables      16     -none-     list     
## levels       4     -none-     character
## call         5     -none-     call     
## x           16     data.frame list     
## usekernel    1     -none-     logical  
## varnames    16     -none-     character
## xNames      16     -none-     character
## problemType  1     -none-     character
## tuneValue    3     data.frame list     
## obsLevels    4     -none-     character
## param        0     -none-     list
# Predict outcome using DryBean_TDA_PC_5.50.5_n4_NbFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_PC_5.50.5_n4_NbFit0, newdata= Dry_Bean_DatasetTest)
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 5
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 6
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 7
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 8
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 9
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 10
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 11
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 12
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 13
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 14
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 15
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 16
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 17
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 18
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 19
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 20
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 21
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 22
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 23
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 24
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 25
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 26
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 27
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 28
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 29
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 30
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 31
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 32
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 33
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 34
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 35
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 36
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 37
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 38
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 39
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 46
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 47
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 48
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 49
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 50
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 51
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 52
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 53
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 54
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 55
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 56
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 57
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 58
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 59
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 60
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 61
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 62
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 63
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 64
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 65
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 66
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 67
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 68
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 69
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 70
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 71
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 72
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 73
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 74
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 75
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 76
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 77
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 78
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 79
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 80
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 81
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 82
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 83
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 84
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 85
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 86
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 87
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 88
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 89
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 90
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 91
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 92
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 93
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 94
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 95
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 96
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 97
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 98
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 99
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 100
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 102
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 106
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 107
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 108
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 112
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 113
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 114
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 115
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 116
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 117
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 118
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 119
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 120
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 122
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 123
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 124
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 125
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 126
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 128
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 129
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 130
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 131
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 133
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 135
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 136
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 138
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 140
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 141
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 144
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 147
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 148
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 161
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 162
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 163
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 164
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 166
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 168
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 171
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 172
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 173
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 175
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 177
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 178
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 179
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 181
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 182
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 184
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 185
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 188
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 190
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 199
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 201
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 202
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 203
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 204
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 205
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 210
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 213
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 215
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 216
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 219
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 220
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 221
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 222
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 223
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 225
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 226
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 229
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 230
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 232
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 233
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 238
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 242
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 244
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 245
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 246
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 247
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 248
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 251
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 252
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 259
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 260
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 261
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 262
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 263
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 265
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 266
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 270
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 303
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 308
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 309
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 310
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 311
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 312
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 315
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 316
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 318
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 319
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 321
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 322
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 324
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 325
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 326
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 327
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 329
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 331
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 332
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 334
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 336
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 337
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 338
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 339
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 341
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 344
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 345
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 347
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 349
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 362
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 363
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 364
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 365
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 366
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 374
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 375
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 377
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 379
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 385
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 387
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 390
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 392
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 394
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 400
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 402
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 406
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 411
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 451
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 475
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 481
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 497
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 498
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 504
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 508
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 509
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 514
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 515
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 519
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 520
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 521
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 523
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 524
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 531
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 533
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 535
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 536
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 543
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 544
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 546
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 547
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 549
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 551
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 552
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 553
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 555
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 558
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 559
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 560
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 561
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 562
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 563
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 566
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 569
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 570
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 574
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 576
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 580
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 581
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 582
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 584
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 585
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 587
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 589
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 590
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 591
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 592
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 593
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 594
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 595
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 598
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 599
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 601
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 602
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 603
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 604
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 605
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 606
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 607
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 612
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 613
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 614
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 616
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 618
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 621
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 627
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 633
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 634
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 635
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 638
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 640
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 641
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 642
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 643
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 644
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 795
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 796
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 803
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 818
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 827
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 828
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 830
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 832
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 834
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 839
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 845
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 847
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 848
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 849
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 856
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 860
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 861
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 863
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 871
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 873
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 884
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 893
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 913
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 927
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 934
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 935
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 951
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 959
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 964
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 981
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1138
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1148
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1161
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1162
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1163
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1164
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1166
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1168
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1171
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1172
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1178
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1179
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1182
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1184
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1185
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1188
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1190
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1199
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1201
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1202
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1204
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1205
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1213
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1706
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1744
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1798
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1805
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1926
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2081
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2229
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2230
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2232
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2233
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2238
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2242
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2244
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2245
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2246
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2247
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2248
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2251
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2252
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2259
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2260
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2261
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2262
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2263
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2265
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2266
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2270
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2303
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2308
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2309
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2310
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2311
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2312
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2315
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2316
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2318
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2319
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2321
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2322
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2324
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2325
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2326
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2327
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2329
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2331
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2332
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2334
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2336
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2337
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2338
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2339
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2341
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2344
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2345
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2347
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2349
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2362
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2363
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2364
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2365
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2366
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2374
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2375
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2377
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2379
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2385
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2387
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2390
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2392
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2394
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2400
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2402
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2406
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2411
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2451
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2475
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2481
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2497
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2498
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2504
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2508
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2509
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2514
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2515
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2519
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2520
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2521
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2523
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2524
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2531
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2533
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2535
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2536
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2543
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2544
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2546
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2547
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2549
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2551
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2552
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2553
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2555
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2558
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2559
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2560
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2561
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2562
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2563
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2566
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2569
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2570
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2574
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2576
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2580
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2581
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2582
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2584
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2585
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2587
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2589
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2590
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2591
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2592
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2593
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2594
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2595
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2598
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2599
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2601
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2602
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2603
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2604
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2605
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2606
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2607
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2612
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2613
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2614
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2616
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2618
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2621
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2627
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2633
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2634
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2635
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2638
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2640
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2641
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2642
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2643
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2644
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2706
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2744
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2795
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2796
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2798
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2800
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2802
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2803
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2805
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2807
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2808
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2809
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2812
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2814
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2816
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2818
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2819
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2823
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2825
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2826
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2827
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2828
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2830
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2832
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2833
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2834
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2838
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2839
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2840
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2841
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2842
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2843
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2844
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2845
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2846
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2847
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2848
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2849
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2851
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2852
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2854
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2855
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2856
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2857
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2859
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2860
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2861
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2863
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2864
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2865
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2866
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2868
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2869
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2871
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2872
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2873
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2875
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2879
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2881
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2882
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2884
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2886
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2888
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2890
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2892
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2893
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2894
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2896
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2897
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2905
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2910
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2912
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2913
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2914
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2915
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2916
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2918
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2920
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2922
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2923
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2924
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2926
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2927
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2929
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2930
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2931
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2932
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2933
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2934
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2935
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2936
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2938
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2939
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2941
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2942
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2943
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2944
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2945
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2946
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2947
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2948
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2949
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2950
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2951
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2952
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2955
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2957
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2959
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2960
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2961
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2962
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2963
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2964
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2965
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2967
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2968
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2969
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2972
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2973
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2974
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2975
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2976
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2977
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2978
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2979
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2980
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2981
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2982
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2983
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2987
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2988
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2989
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2990
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2991
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2993
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2994
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2995
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2996
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2997
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2998
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2999
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3000
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3001
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3002
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3008
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3009
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3013
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3014
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3015
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3017
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3019
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3020
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3021
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3022
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3024
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3025
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3027
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3029
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3030
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3031
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3032
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3033
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3034
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3035
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3036
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3037
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3038
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3039
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3040
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3041
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3042
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3043
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3045
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3046
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3047
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3048
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3049
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3053
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3055
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3056
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3057
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3058
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3059
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3060
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3061
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3062
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3063
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3064
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3065
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3066
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3067
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3068
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3069
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3070
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3071
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3072
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3073
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3075
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3076
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3077
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3079
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3080
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3081
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3082
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3083
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3084
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3085
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3086
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3087
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3088
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3089
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3090
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3091
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3092
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3093
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3094
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3095
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3096
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3097
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3098
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3099
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3100
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3102
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3106
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3107
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3108
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3112
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3113
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3114
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3115
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3116
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3117
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3118
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3119
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3120
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3122
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3123
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3124
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3125
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3126
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3128
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3129
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3130
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3131
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3133
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3135
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3136
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3138
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3140
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3141
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3144
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3147
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3148
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3161
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3162
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3163
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3164
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3166
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3168
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3171
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3172
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3173
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3175
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3177
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3178
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3179
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3181
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3182
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3184
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3185
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3188
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3190
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3199
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3201
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3202
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3203
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3204
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3205
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3210
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3213
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3215
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3216
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3219
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3220
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3221
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3222
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3223
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3225
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3226
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3229
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3230
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3232
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3233
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3238
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3242
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3244
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3245
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3246
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3247
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3248
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3251
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3252
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3259
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3260
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3261
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3262
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3263
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3265
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3266
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3270
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3303
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3308
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3309
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3310
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3311
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3312
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3315
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3316
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3318
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3319
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3321
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3322
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3324
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3325
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3326
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3327
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3329
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3331
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3332
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3334
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3336
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3337
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3338
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3339
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3341
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3344
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3345
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3347
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3349
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3362
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3363
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3364
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3365
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3366
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3374
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3375
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3377
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3379
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3385
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3387
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3390
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3392
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3394
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3400
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3402
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3406
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3411
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3451
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3475
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3481
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3497
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3498
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3504
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3508
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3509
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3514
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3515
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3519
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3520
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3521
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3523
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3524
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3531
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3533
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3535
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3536
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3543
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3544
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3546
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3547
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3549
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3551
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3552
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3553
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3555
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3558
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3559
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3560
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3561
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3562
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3563
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3566
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3569
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3570
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3574
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3576
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3580
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3581
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3582
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3584
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3585
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3587
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3589
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3590
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3591
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3592
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3593
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3594
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3595
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3598
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3599
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3601
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3602
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3603
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3604
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3605
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3606
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3607
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3612
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3613
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3614
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3616
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3618
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3621
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3627
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3633
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3634
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3635
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3638
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3640
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3641
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3642
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3643
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3644
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3706
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3744
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3795
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3796
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3798
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3800
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3802
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3803
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3805
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3807
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3808
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3809
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3812
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3814
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3816
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3818
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3819
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3823
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3825
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3826
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3827
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3828
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3830
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3832
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3833
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3834
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3838
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3839
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3840
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3841
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3842
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3843
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3844
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3845
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3846
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3847
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3848
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3849
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3851
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3852
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3854
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3855
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3856
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3857
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3859
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3860
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3861
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3863
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3864
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3865
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3866
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3868
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3869
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3871
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3872
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3873
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3875
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3879
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3881
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3882
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3884
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3886
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3888
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3890
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3892
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3893
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3894
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3896
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3897
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3905
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3910
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3912
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3913
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3914
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3915
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3916
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3918
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3920
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3922
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3923
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3924
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3926
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3927
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3929
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3930
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3931
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3932
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3933
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3934
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3935
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3936
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3938
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3939
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3941
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3942
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3943
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3944
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3945
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3946
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3947
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3948
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3949
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3950
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3951
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3952
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3955
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3957
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3959
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3960
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3961
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3962
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3963
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3964
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3965
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3967
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3968
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3969
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3972
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3973
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3974
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3975
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3976
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3977
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3978
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3979
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3980
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3981
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3982
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3983
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3987
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3988
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3989
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3990
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3991
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3993
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3994
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3995
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3996
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3997
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3998
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3999
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4000
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4001
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4002
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4008
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4009
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4013
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4014
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4015
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4017
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4019
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4020
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4021
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4022
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4024
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4025
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4027
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4029
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4030
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4031
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4032
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4033
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4034
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4035
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4036
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4037
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4038
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4039
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4040
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4041
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4042
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4043
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4045
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4046
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4047
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4048
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4049
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4053
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4055
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4056
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4057
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4058
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4059
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4060
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4061
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4062
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4063
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4064
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4065
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4066
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4067
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4068
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4069
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4070
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4071
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4072
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4073
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4075
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4076
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4077
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4079
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4080
# Create confusion matrix to assess model fit/performance on test data
db_tda_pc_5.50.5_n4_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
db_tda_pc_5.50.5_n4_db_nb_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   81      330     1   600  314
##   BOMBAY          1    156    0        0     0     8    0
##   CALI           18      0  386        0     8     0    1
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ          13      0   22      733   569     0  475
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3615          
##                  95% CI : (0.3468, 0.3765)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2771          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000     0.78937          0.0000
## Specificity                  0.64007       0.99771     0.99248          1.0000
## Pos Pred Value               0.21538       0.94545     0.93462             NaN
## Neg Pred Value               0.98661       1.00000     0.97191          0.7395
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.08922       0.03824     0.09461          0.0000
## Detection Prevalence         0.41422       0.04044     0.10123          0.0000
## Balanced Accuracy            0.77963       0.99885     0.89092          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9844        0.000      0.0000
## Specificity                0.6451        1.000      1.0000
## Pos Pred Value             0.3140          NaN         NaN
## Neg Pred Value             0.9960        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1395        0.000      0.0000
## Detection Prevalence       0.4441        0.000      0.0000
## Balanced Accuracy          0.8147        0.500      0.5000
db_tda_pc_5.50.5_n4_db_nb_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   81      330     1   600  314
##   BOMBAY          1    156    0        0     0     8    0
##   CALI           18      0  386        0     8     0    1
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ          13      0   22      733   569     0  475
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3615          
##                  95% CI : (0.3468, 0.3765)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2771          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000     0.78937          0.0000
## Specificity                  0.64007       0.99771     0.99248          1.0000
## Pos Pred Value               0.21538       0.94545     0.93462             NaN
## Neg Pred Value               0.98661       1.00000     0.97191          0.7395
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.08922       0.03824     0.09461          0.0000
## Detection Prevalence         0.41422       0.04044     0.10123          0.0000
## Balanced Accuracy            0.77963       0.99885     0.89092          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9844        0.000      0.0000
## Specificity                0.6451        1.000      1.0000
## Pos Pred Value             0.3140          NaN         NaN
## Neg Pred Value             0.9960        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1395        0.000      0.0000
## Detection Prevalence       0.4441        0.000      0.0000
## Balanced Accuracy          0.8147        0.500      0.5000
db_tda_pc_5.50.5_n4_db_nb_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   3.615196e-01   2.770842e-01   3.467586e-01   3.764792e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##   6.574974e-46            NaN
db_tda_pc_5.50.5_n4_db_nb_cf0_ov_acc<-db_tda_pc_5.50.5_n4_db_nb_cf0$overall[1]
db_tda_pc_5.50.5_n4_db_nb_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9191919   0.6400651      0.2153846      0.9866109 0.2153846
## Class: BOMBAY     1.0000000   0.9977064      0.9454545      1.0000000 0.9454545
## Class: CALI       0.7893661   0.9924812      0.9346247      0.9719116 0.9346247
## Class: DERMASON   0.0000000   1.0000000            NaN      0.7394608        NA
## Class: HOROZ      0.9844291   0.6450600      0.3140177      0.9960317 0.3140177
## Class: SEKER      0.0000000   1.0000000            NaN      0.8509804        NA
## Class: SIRA       0.0000000   1.0000000            NaN      0.8063725        NA
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9191919 0.3489933 0.09705882     0.08921569
## Class: BOMBAY   1.0000000 0.9719626 0.03823529     0.03823529
## Class: CALI     0.7893661 0.8558758 0.11985294     0.09460784
## Class: DERMASON 0.0000000        NA 0.26053922     0.00000000
## Class: HOROZ    0.9844291 0.4761506 0.14166667     0.13946078
## Class: SEKER    0.0000000        NA 0.14901961     0.00000000
## Class: SIRA     0.0000000        NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.41421569         0.7796285
## Class: BOMBAY             0.04044118         0.9988532
## Class: CALI               0.10122549         0.8909236
## Class: DERMASON           0.00000000         0.5000000
## Class: HOROZ              0.44411765         0.8147445
## Class: SEKER              0.00000000         0.5000000
## Class: SIRA               0.00000000         0.5000000
db_tda_pc_5.50.5_n4_db_nb_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n4_db_nb_cf0$byClass[5:7]


###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_pca_5.50.5_nb_n4_3_fold<-(db_nb_fit_re - db_tda_pc_5.50.5_n4_nb_fit_re)
diff_drybean_tda_pca_5.50.5_nb_n4_3_fold
##      Accuracy
## 1 -0.05973130
## 2 -0.05858321
## 3 -0.04956963
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nb.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n4_3_fold),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nb.n4_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nb.n4_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n4_3_fold$probLeft/bst_dbf_db_tda_pca_5.50.5_nb.n4_3_fold$probRight
bst_dbf_db_tda_pca_5.50.5_nb.n4_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nb.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n4_3_fold),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nb.n4_3_fold
## $winLeft
## [1] 0.9901667
## 
## $winRope
## [1] 0.009833333
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nb.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n4_3_fold),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nb.n4_3_fold
## $left
## [1] 0.9967735
## 
## $rope
## [1] 0.001652132
## 
## $right
## [1] 0.001574345
# Rope Plot
plot(rope(diff_drybean_tda_pca_5.50.5_nb_n4_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb_n4_3_fold))
#bf_tda_pca_5.50.5_nb.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_pca_5.50.5_nb_n4_3_fold)
## t = -17.417, df = 2, p-value = 0.00328
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.06978587 -0.04213689
## sample estimates:
##   mean of x 
## -0.05596138
### Test set diff
diff_drybean_tda_pca_5.50.5_nb.n4_test<-(db_nb_cf_ov_acc - db_tda_pc_5.50.5_n4_db_nb_cf0_ov_acc)
diff_drybean_tda_pca_5.50.5_nb.n4_test
##  Accuracy 
## 0.5328431
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_dbf_db_tda_pca_5.50.5_nb.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n4_test),-0.01,0.01)
bst_dbf_db_tda_pca_5.50.5_nb.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_dbf_db_tda_pca_5.50.5_nb.n4_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n4_test$probLeft/bst_dbf_db_tda_pca_5.50.5_nb.n4_test$probRight
bst_dbf_db_tda_pca_5.50.5_nb.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_dbf_db_tda_pca_5.50.5_nb.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n4_test),-0.01,0.01)
bsr_dbf_db_tda_pca_5.50.5_nb.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.1589333
## 
## $winRight
## [1] 0.8410667
# Bayesian Correlated Test

bct_dbf_db_tda_pca_5.50.5_nb.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n4_test),0.1,-0.01,0.01)
bct_dbf_db_tda_pca_5.50.5_nb.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nb.n4_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb.n4_test)) #bf_tda_pca_5.50.5_nb.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n4_test))

##Node5

#DryBean_TDA_PC_5.50.5_n5_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n5.vec, 
 #               method = 'nb', 
 #              trControl = fitControl,
 #               metric='Accuracy')

#DryBean_TDA_PC_5.50.5_n5_NbFit0
#DryBean_TDA_PC_5.50.5_n5_NbFit0$resample
#db_tda_pc_5.50.5_n5_nb_fit_re<-DryBean_TDA_PC_5.50.5_n5_NbFit0$resample[1]

#summary(DryBean_TDA_PC_5.50.5_n5_NbFit0)

# Predict outcome using DryBean_TDA_PC_5.50.5_n5_NbFit0 from training data based on testing data
#pred0 <- predict(DryBean_TDA_PC_5.50.5_n5_NbFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
#db_tda_pc_5.50.5_n5_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
#db_tda_pc_5.50.5_n5_db_nb_cf0
#db_tda_pc_5.50.5_n5_db_nb_cf0 
#db_tda_pc_5.50.5_n5_db_nb_cf0$overall
#db_tda_pc_5.50.5_n5_db_nb_cf0_ov_acc<-db_tda_pc_5.50.5_n5_db_nb_cf0$overall[1]
#db_tda_pc_5.50.5_n5_db_nb_cf0$byClass
#db_tda_pc_5.50.5_n5_db_nb_cf0_pre_rec_f1<-db_tda_pc_5.50.5_n5_db_nb_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

#diff_drybean_tda_pca_5.50.5_nb_n5_3_fold<-(db_nb_fit_re - db_tda_pc_5.50.5_n5_nb_fit_re)
#diff_drybean_tda_pca_5.50.5_nb_n5_3_fold

## Bayesian Tests 3-fold diff

# Bayesian Sign Test

#bst_dbf_db_tda_pca_5.50.5_nb.n5_3_fold<-#BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n5_3_fold),-0.01,0.01)
#bst_dbf_db_tda_pca_5.50.5_nb.n5_3_fold

# Odds Left Bayesian Sign Test 

#bst_dbf_db_tda_pca_5.50.5_nb.n5_3_fold_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n5_3_fold$probLeft/#bst_dbf_db_tda_pca_5.50.5_nb.n5_3_fold$probRight
#bst_dbf_db_tda_pca_5.50.5_nb.n5_3_fold_odds.left

# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_nb.n5_3_fold<-#BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n4_3_fold),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_nb.n5_3_fold

# Bayesian Correlated Test

#bct_dbf_db_tda_pca_5.50.5_nb.n5_3_fold<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n5_3_fold),0.1,-0.01,0.01)
#bct_dbf_db_tda_pca_5.50.5_nb.n5_3_fold

# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nb_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb_n5_3_fold))
#bf_tda_pca_5.50.5_nb.n5_3_fold

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb_n5_3_fold))


### Test set diff
#diff_drybean_tda_pca_5.50.5_nb.n5_test<-(db_nb_cf_ov_acc - db_tda_pc_5.50.5_n5_db_nb_cf0_ov_acc)
#diff_drybean_tda_pca_5.50.5_nb.n5_test


## Bayesian Tests Test set diff

# Bayesian Sign Test

#bst_dbf_db_tda_pca_5.50.5_nb.n5_test<-#BayesianSignTest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n5_test),-0.01,0.01)
#bst_dbf_db_tda_pca_5.50.5_nb.n5_test

# Odds Left Bayesian Sign Test 

#bst_dbf_db_tda_pca_5.50.5_nb.n5_test_odds.left<-bst_dbf_db_tda_pca_5.50.5_nb.n5_test$probLeft/#bst_dbf_db_tda_pca_5.50.5_nb.n5_test$probRight
#bst_dbf_db_tda_pca_5.50.5_nb.n5_test_odds.left

# Bayesian Signed Rank Test

#bsr_dbf_db_tda_pca_5.50.5_nb.n5_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n5_test),-0.01,0.01)
#bsr_dbf_db_tda_pca_5.50.5_nb.n5_test

# Bayesian Correlated Test

#bct_dbf_db_tda_pca_5.50.5_nb.n5_test<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n5_test),0.1,-0.01,0.01)
#bct_dbf_db_tda_pca_5.50.5_nb.n5_test

# Rope Plot
#plot(rope(diff_drybean_tda_pca_5.50.5_nb.n5_test)))

#BayesFactor
#bf_tda_pca_5.50.5_nb.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_pca_5.50.5_nb.n5_test)) #bf_tda_pca_5.50.5_nb.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_pca_5.50.5_nb.n5_test))

##With TDA KDE filter 5 intervals, 50% overlap, 5 bins 
##Node1

#DryBean_TDA_KDE_5.50.5_n1_NbFit0 <- train(as.factor(Class) ~ ., data = #tda.m_kde_dry_bean_dataset_5.50.5.n5.n1.vec, 
#                method = 'nb', 
#                trControl = fitControl,
#                metric='Accuracy')

#DryBean_TDA_KDE_5.50.5_n1_NbFit0
#DryBean_TDA_KDE_5.50.5_n1_NbFit0$resample
#nb_tda_kde_5.50.5_n1_nb_fit_re<-DryBean_TDA_KDE_5.50.5_n1_NbFit0$resample[1]

#summary(DryBean_TDA_KDE_5.50.5_n1_NbFit0)

# Predict outcome using DryBean_TDA_KDE_5.50.5_n1_NbFit0 from training data based on testing data
#pred0 <- predict(DryBean_TDA_KDE_5.50.5_n1_NbFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
#nb_tda_kde_5.50.5_n1_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
#nb_tda_kde_5.50.5_n1_db_nb_cf0
#nb_tda_kde_5.50.5_n1_db_nb_cf0 
#nb_tda_kde_5.50.5_n1_db_nb_cf0$overall
#nb_tda_kde_5.50.5_n1_db_nb_cf0_ov_acc<-nb_tda_kde_5.50.5_n1_db_nb_cf0$overall[1]
#nb_tda_kde_5.50.5_n1_db_nb_cf0$byClas1
#nb_tda_kde_5.50.5_n1_db_nb_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n1_db_nb_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

#diff_drybean_tda_kde_5.50.5_nb_n1_3_fold<-(db_nb_fit_re - nb_tda_kde_5.50.5_n1_nb_fit_re)
#diff_drybean_tda_kde_5.50.5_nb_n1_3_fold

## Bayesian Tests 3-fold diff

# Bayesian Sign Test

#bst_tda_kde_5.50.5_nb.n1_3_fold<-#BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n1_3_fold),-0.01,0.01)
#bst_tda_kde_5.50.5_nb.n1_3_fold

# Odds Left Bayesian Sign Test 

#bst_tda_kde_5.50.5_nb.n1_3_fold_odds.left<-bst_tda_kde_5.50.5_nb.n1_3_fold$probLeft/#bst_tda_kde_5.50.5_nb.n1_3_fold$probRight
#bst_tda_kde_5.50.5_nb.n1_3_fold_odds.left

# Bayesian Signed Rank Test

#bsr_tda_kde_5.50.5_nb.n1_3_fold<-#BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n1_3_fold),-0.01,0.01)
#bsr_tda_kde_5.50.5_nb.n1_3_fold

# Bayesian Correlated Test

bct_tda_kde_5.50.5_nb.n1_3_fold<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n1_3_fold),0.1,-0.01,0.01)
#bct_tda_kde_5.50.5_nb.n1_3_fold

# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb_n1_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n1_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb_n1_3_fold))
#bf_tda_kde_5.50.5_nb.n1_3_fold

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n1_3_fold))


### Test set diff
#diff_drybean_tda_kde_5.50.5_nb.n1_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n1_db_nb_cf0_ov_acc)
#diff_drybean_tda_kde_5.50.5_nb.n1_test


## Bayesian Tests Test set diff

# Bayesian Sign Test

#bst_tda_kde_5.50.5_nb.n1_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n1_test),-0.01,0.01)
#bst_tda_kde_5.50.5_nb.n1_test

# Odds Left Bayesian Sign Test 

#bst_tda_kde_5.50.5_nb.n1_test_odds.left<-bst_tda_kde_5.50.5_nb.n1_test$probLeft/#bst_tda_kde_5.50.5_nb.n1_test$probRight
#bst_tda_kde_5.50.5_nb.n1_test_odds.left

# Bayesian Signed Rank Test

#bsr_tda_kde_5.50.5_nb.n1_test<-#BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n1_test),-0.01,0.01)
#bsr_tda_kde_5.50.5_nb.n1_test

# Bayesian Correlated Test

#bct_tda_kde_5.50.5_nb.n1_test<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n1_test),0.1,-0.01,0.01)
#bct_tda_kde_5.50.5_nb.n1_test

# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb.n1_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n1_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb.n1_test)) #bf_tda_pca_5.50.5_nb.n1_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n1_test))

##Node2

#DryBean_TDA_KDE_5.50.5_n2_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n2.vec, 
#                method = 'nb', 
#                trControl = fitControl,
#                metric='Accuracy')

#DryBean_TDA_KDE_5.50.5_n2_NbFit0
#DryBean_TDA_KDE_5.50.5_n2_NbFit0$resample
#nb_tda_kde_5.50.5_n2_nb_fit_re<-DryBean_TDA_KDE_5.50.5_n2_NbFit0$resample[1]

#summary(DryBean_TDA_KDE_5.50.5_n2_NbFit0)

# Predict outcome using DryBean_TDA_KDE_5.50.5_n2_NbFit0 from training data based on testing data
#pred0 <- predict(DryBean_TDA_KDE_5.50.5_n2_NbFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
#nb_tda_kde_5.50.5_n2_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
#nb_tda_kde_5.50.5_n2_db_nb_cf0
#nb_tda_kde_5.50.5_n2_db_nb_cf0 
#nb_tda_kde_5.50.5_n2_db_nb_cf0$overall
#nb_tda_kde_5.50.5_n2_db_nb_cf0_ov_acc<-nb_tda_kde_5.50.5_n2_db_nb_cf0$overall[1]
#nb_tda_kde_5.50.5_n2_db_nb_cf0$byClass
#nb_tda_kde_5.50.5_n2_db_nb_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n2_db_nb_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

#diff_drybean_tda_kde_5.50.5_nb_n2_3_fold<-(db_nb_fit_re - nb_tda_kde_5.50.5_n2_nb_fit_re)
#diff_drybean_tda_kde_5.50.5_nb_n2_3_fold

## Bayesian Tests 3-fold diff

# Bayesian Sign Test

#bst_tda_kde_5.50.5_nb.n2_3_fold<-#BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n2_3_fold),-0.01,0.01)
#bst_tda_kde_5.50.5_nb.n2_3_fold

# Odds Left Bayesian Sign Test 

#bst_tda_kde_5.50.5_nb.n2_3_fold_odds.left<-bst_tda_kde_5.50.5_nb.n2_3_fold$probLeft/#bst_tda_kde_5.50.5_nb.n2_3_fold$probRight
#bst_tda_kde_5.50.5_nb.n2_3_fold_odds.left

# Bayesian Signed Rank Test

#bsr_tda_kde_5.50.5_nb.n2_3_fold<-#BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n2_3_fold),-0.01,0.01)
#bsr_tda_kde_5.50.5_nb.n2_3_fold

# Bayesian Correlated Test

#bct_tda_kde_5.50.5_nb.n2_3_fold<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n2_3_fold),0.1,-0.01,0.01)
#bct_tda_kde_5.50.5_nb.n2_3_fold

# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb_n2_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n2_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb_n2_3_fold))
#bf_tda_kde_5.50.5_nb.n2_3_fold

#t_test
t#.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n2_3_fold))


### Test set diff
#diff_drybean_tda_kde_5.50.5_nb.n2_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n2_db_nb_cf0_ov_acc)
#diff_drybean_tda_kde_5.50.5_nb.n2_test


## Bayesian Tests Test set diff

# Bayesian Sign Test

#bst_tda_kde_5.50.5_nb.n2_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n2_test),-0.01,0.01)
#bst_tda_kde_5.50.5_nb.n2_test

# Odds Left Bayesian Sign Test 

#bst_tda_kde_5.50.5_nb.n2_test_odds.left<-bst_tda_kde_5.50.5_nb.n2_test$probLeft/#bst_tda_kde_5.50.5_nb.n2_test$probRight
#bst_tda_kde_5.50.5_nb.n2_test_odds.left

# Bayesian Signed Rank Test

#bsr_tda_kde_5.50.5_nb.n2_test<-#BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n2_test),-0.01,0.01)
#bsr_tda_kde_5.50.5_nb.n2_test

# Bayesian Correlated Test

#bct_tda_kde_5.50.5_nb.n2_test<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n2_test),0.1,-0.01,0.01)
#bct_tda_kde_5.50.5_nb.n2_test

# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb.n2_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n2_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb.n2_test)) #bf_tda_kde_5.50.5_nb.n2_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n2_test))

##Node3

#DryBean_TDA_KDE_5.50.5_n3_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n3.vec, 
#                method = 'nb', 
 #               trControl = fitControl,
  #              metric='Accuracy')

#DryBean_TDA_KDE_5.50.5_n3_NbFit0
#DryBean_TDA_KDE_5.50.5_n3_NbFit0$resample
#nb_tda_kde_5.50.5_n3_nb_fit_re<-DryBean_TDA_KDE_5.50.5_n3_NbFit0$resample[1]

#summary(DryBean_TDA_KDE_5.50.5_n3_NbFit0)

# Predict outcome using DryBean_TDA_KDE_5.50.5_n3_NbFit0 from training data based on testing data
#pred0 <- predict(DryBean_TDA_KDE_5.50.5_n3_NbFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
#nb_tda_kde_5.50.5_n3_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
#nb_tda_kde_5.50.5_n3_db_nb_cf0
##nb_tda_kde_5.50.5_n3_db_nb_cf0 
#nb_tda_kde_5.50.5_n3_db_nb_cf0$overall
#nb_tda_kde_5.50.5_n3_db_nb_cf0_ov_acc<-nb_tda_kde_5.50.5_n3_db_nb_cf0$overall[1]
#nb_tda_kde_5.50.5_n3_db_nb_cf0$byClass
#nb_tda_kde_5.50.5_n3_db_nb_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n3_db_nb_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

#diff_drybean_tda_kde_5.50.5_nb_n3_3_fold<-(db_nb_fit_re - nb_tda_kde_5.50.5_n3_nb_fit_re)
#diff_drybean_tda_kde_5.50.5_nb_n3_3_fold

## Bayesian Tests 3-fold diff

# Bayesian Sign Test

#bst_tda_kde_5.50.5_nb.n3_3_fold<-#BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n3_3_fold),-0.01,0.01)
#bst_tda_kde_5.50.5_nb.n3_3_fold

# Odds Left Bayesian Sign Test 

#bst_tda_kde_5.50.5_nb.n3_3_fold_odds.left<-bst_tda_kde_5.50.5_nb.n3_3_fold$probLeft/#bst_tda_kde_5.50.5_nb.n3_3_fold$probRight
#bst_tda_kde_5.50.5_nb.n3_3_fold_odds.left

# Bayesian Signed Rank Test

#bsr_tda_kde_5.50.5_nb.n3_3_fold<-#BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n3_3_fold),-0.01,0.01)
#bsr_tda_kde_5.50.5_nb.n3_3_fold

# Bayesian Correlated Test

#bct_tda_kde_5.50.5_nb.n3_3_fold<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n3_3_fold),0.1,-0.01,0.01)
#bct_tda_kde_5.50.5_nb.n3_3_fold

# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb_n3_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n3_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb_n3_3_fold))
#bf_tda_kde_5.50.5_nb.n3_3_fold

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n3_3_fold))


### Test set diff
#diff_drybean_tda_kde_5.50.5_nb.n3_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n3_db_nb_cf0_ov_acc)
#diff_drybean_tda_kde_5.50.5_nb.n3_test


## Bayesian Tests Test set diff

# Bayesian Sign Test

#bst_tda_kde_5.50.5_nb.n3_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n3_test),-0.01,0.01)
#bst_tda_kde_5.50.5_nb.n3_test

# Odds Left Bayesian Sign Test 

#bst_tda_kde_5.50.5_nb.n3_test_odds.left<-bst_tda_kde_5.50.5_nb.n3_test$probLeft/#bst_tda_kde_5.50.5_nb.n3_test$probRight
#bst_tda_kde_5.50.5_nb.n3_test_odds.left

# Bayesian Signed Rank Test

#bsr_tda_kde_5.50.5_nb.n3_test<-#BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n3_test),-0.01,0.01)
#bsr_tda_kde_5.50.5_nb.n3_test

# Bayesian Correlated Test

#bct_tda_kde_5.50.5_nb.n3_test<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n3_test),0.1,-0.01,0.01)
#bct_tda_kde_5.50.5_nb.n3_test

# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb.n3_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n3_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb.n3_test)) #bf_tda_kde_5.50.5_nb.n3_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n3_test))


##Node4

DryBean_TDA_KDE_5.50.5_n4_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n4.vec, 
                method = 'nb', 
                trControl = fitControl,
                metric='Accuracy')
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 9
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 33
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 35
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 52
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 60
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 77
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 90
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 97
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 221
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 33
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 35
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 46
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 52
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 60
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 77
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 90
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 97
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 126
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 221
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 338
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 377
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 52
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 95
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 120
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 131
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 52
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 54
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 95
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 120
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 131
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 177
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 23
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 47
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 50
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 51
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 66
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 73
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 113
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 47
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 49
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 50
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 51
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 54
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 66
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 73
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 113
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 144
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 441
DryBean_TDA_KDE_5.50.5_n4_NbFit0
## Naive Bayes 
## 
## 1590 samples
##   16 predictor
##    4 classes: 'BARBUNYA', 'BOMBAY', 'CALI', 'HOROZ' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 1060, 1059, 1061 
## Resampling results across tuning parameters:
## 
##   usekernel  Accuracy   Kappa    
##   FALSE      0.9565835  0.9373926
##    TRUE      0.9540749  0.9334847
## 
## Tuning parameter 'fL' was held constant at a value of 0
## Tuning
##  parameter 'adjust' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were fL = 0, usekernel = FALSE and adjust
##  = 1.
DryBean_TDA_KDE_5.50.5_n4_NbFit0$resample
##    Accuracy     Kappa Resample
## 1 0.9622642 0.9452660    Fold1
## 2 0.9698682 0.9563098    Fold2
## 3 0.9376181 0.9106021    Fold3
nb_tda_kde_5.50.5_n4_nb_fit_re<-DryBean_TDA_KDE_5.50.5_n4_NbFit0$resample[1]

summary(DryBean_TDA_KDE_5.50.5_n4_NbFit0)
##             Length Class      Mode     
## apriori      4     table      numeric  
## tables      16     -none-     list     
## levels       4     -none-     character
## call         5     -none-     call     
## x           16     data.frame list     
## usekernel    1     -none-     logical  
## varnames    16     -none-     character
## xNames      16     -none-     character
## problemType  1     -none-     character
## tuneValue    3     data.frame list     
## obsLevels    4     -none-     character
## param        0     -none-     list
# Predict outcome using DryBean_TDA_KDE_5.50.5_n4_NbFit0 from training data based on testing data
pred0 <- predict(DryBean_TDA_KDE_5.50.5_n4_NbFit0, newdata= Dry_Bean_DatasetTest)
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 5
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 6
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 7
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 8
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 9
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 10
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 11
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 12
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 13
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 14
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 15
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 16
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 17
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 18
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 19
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 20
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 21
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 22
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 23
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 24
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 25
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 26
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 27
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 28
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 29
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 30
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 31
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 32
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 33
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 34
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 35
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 36
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 37
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 38
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 39
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 40
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 41
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 42
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 43
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 44
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 45
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 46
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 47
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 48
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 49
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 50
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 51
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 52
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 53
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 54
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 55
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 56
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 57
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 58
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 59
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 60
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 61
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 62
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 63
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 64
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 65
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 66
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 67
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 68
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 69
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 70
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 71
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 72
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 73
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 74
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 75
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 76
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 77
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 78
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 79
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 80
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 81
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 82
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 83
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 84
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 85
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 86
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 87
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 88
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 89
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 90
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 91
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 92
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 93
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 94
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 95
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 96
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 97
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 98
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 99
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 100
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 102
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 106
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 107
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 108
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 112
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 113
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 114
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 115
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 116
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 117
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 118
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 119
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 120
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 122
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 123
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 124
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 125
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 126
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 128
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 129
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 130
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 131
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 133
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 135
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 136
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 138
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 140
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 141
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 144
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 147
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 148
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 161
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 162
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 163
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 164
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 166
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 168
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 171
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 172
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 173
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 175
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 177
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 178
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 179
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 181
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 182
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 184
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 185
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 188
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 190
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 199
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 201
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 202
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 203
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 204
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 205
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 210
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 213
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 215
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 216
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 219
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 220
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 221
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 222
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 223
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 225
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 226
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 229
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 230
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 232
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 233
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 238
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 242
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 244
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 245
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 246
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 247
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 248
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 251
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 252
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 259
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 260
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 261
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 262
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 263
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 265
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 266
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 270
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 303
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 308
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 309
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 310
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 311
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 312
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 315
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 316
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 318
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 319
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 321
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 322
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 324
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 325
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 326
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 327
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 329
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 331
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 332
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 334
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 336
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 337
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 338
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 339
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 341
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 344
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 345
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 347
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 349
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 362
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 363
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 364
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 365
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 366
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 374
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 375
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 377
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 379
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 385
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 387
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 390
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 392
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 394
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 400
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 402
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 406
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 411
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 451
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 475
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 481
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 497
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 498
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 504
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 508
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 509
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 514
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 515
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 519
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 520
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 521
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 523
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 524
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 531
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 533
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 535
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 536
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 543
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 544
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 546
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 547
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 549
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 551
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 552
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 553
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 555
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 558
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 559
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 560
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 561
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 562
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 563
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 566
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 569
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 570
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 574
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 576
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 580
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 581
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 582
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 584
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 585
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 587
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 589
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 590
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 591
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 592
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 593
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 594
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 595
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 598
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 599
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 601
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 602
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 603
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 604
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 605
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 606
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 607
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 612
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 613
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 614
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 616
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 618
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 621
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 627
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 633
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 634
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 635
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 638
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 640
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 641
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 642
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 643
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 644
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 795
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 796
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 803
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 818
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 827
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 828
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 830
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 832
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 834
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 839
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 845
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 847
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 848
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 849
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 856
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 860
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 861
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 863
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 871
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 873
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 884
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 893
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 913
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 927
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 934
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 935
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 951
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 959
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 964
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 981
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1138
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1148
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1161
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1162
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1163
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1164
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1166
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1168
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1171
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1172
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1178
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1179
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1182
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1184
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1185
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1188
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1190
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1199
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1201
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1202
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1204
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1205
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1213
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1706
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1744
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1798
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1805
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1926
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 1971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2081
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2229
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2230
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2232
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2233
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2238
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2242
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2244
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2245
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2246
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2247
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2248
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2251
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2252
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2259
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2260
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2261
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2262
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2263
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2265
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2266
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2270
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2303
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2308
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2309
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2310
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2311
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2312
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2315
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2316
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2318
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2319
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2321
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2322
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2324
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2325
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2326
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2327
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2329
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2331
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2332
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2334
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2336
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2337
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2338
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2339
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2341
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2344
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2345
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2347
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2349
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2362
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2363
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2364
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2365
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2366
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2374
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2375
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2377
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2379
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2385
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2387
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2390
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2392
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2394
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2400
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2402
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2406
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2411
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2451
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2475
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2481
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2497
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2498
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2504
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2508
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2509
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2514
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2515
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2519
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2520
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2521
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2523
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2524
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2531
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2533
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2535
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2536
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2543
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2544
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2546
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2547
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2549
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2551
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2552
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2553
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2555
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2558
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2559
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2560
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2561
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2562
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2563
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2566
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2569
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2570
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2574
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2576
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2580
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2581
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2582
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2584
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2585
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2587
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2589
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2590
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2591
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2592
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2593
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2594
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2595
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2598
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2599
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2601
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2602
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2603
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2604
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2605
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2606
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2607
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2612
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2613
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2614
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2616
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2618
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2621
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2627
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2633
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2634
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2635
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2638
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2640
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2641
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2642
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2643
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2644
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2706
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2744
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2795
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2796
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2798
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2800
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2802
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2803
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2805
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2807
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2808
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2809
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2812
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2814
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2816
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2818
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2819
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2823
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2825
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2826
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2827
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2828
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2830
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2832
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2833
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2834
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2838
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2839
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2840
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2841
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2842
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2843
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2844
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2845
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2846
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2847
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2848
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2849
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2851
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2852
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2854
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2855
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2856
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2857
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2859
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2860
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2861
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2863
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2864
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2865
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2866
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2868
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2869
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2871
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2872
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2873
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2875
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2879
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2881
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2882
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2884
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2886
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2888
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2890
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2892
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2893
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2894
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2896
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2897
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2905
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2910
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2912
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2913
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2914
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2915
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2916
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2918
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2920
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2922
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2923
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2924
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2926
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2927
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2929
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2930
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2931
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2932
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2933
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2934
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2935
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2936
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2938
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2939
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2941
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2942
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2943
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2944
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2945
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2946
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2947
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2948
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2949
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2950
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2951
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2952
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2955
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2957
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2959
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2960
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2961
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2962
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2963
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2964
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2965
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2967
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2968
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2969
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2972
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2973
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2974
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2975
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2976
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2977
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2978
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2979
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2980
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2981
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2982
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2983
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2987
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2988
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2989
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2990
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2991
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2993
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2994
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2995
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2996
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2997
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2998
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 2999
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3000
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3001
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3002
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3008
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3009
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3013
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3014
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3015
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3017
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3019
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3020
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3021
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3022
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3024
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3025
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3027
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3029
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3030
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3031
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3032
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3033
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3034
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3035
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3036
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3037
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3038
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3039
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3040
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3041
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3042
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3043
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3045
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3046
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3047
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3048
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3049
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3053
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3055
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3056
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3057
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3058
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3059
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3060
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3061
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3062
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3063
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3064
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3065
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3066
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3067
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3068
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3069
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3070
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3071
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3072
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3073
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3075
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3076
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3077
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3079
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3080
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3081
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3082
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3083
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3084
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3085
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3086
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3087
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3088
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3089
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3090
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3091
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3092
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3093
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3094
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3095
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3096
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3097
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3098
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3099
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3100
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3101
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3102
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3103
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3104
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3105
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3106
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3107
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3108
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3109
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3110
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3111
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3112
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3113
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3114
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3115
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3116
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3117
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3118
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3119
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3120
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3121
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3122
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3123
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3124
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3125
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3126
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3127
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3128
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3129
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3130
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3131
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3132
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3133
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3134
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3135
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3136
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3137
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3138
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3139
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3140
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3141
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3142
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3143
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3144
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3145
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3146
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3147
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3148
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3149
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3150
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3151
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3152
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3153
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3154
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3155
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3156
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3157
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3158
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3159
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3160
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3161
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3162
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3163
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3164
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3165
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3166
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3167
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3168
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3169
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3170
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3171
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3172
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3173
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3174
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3175
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3176
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3177
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3178
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3179
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3180
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3181
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3182
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3183
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3184
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3185
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3186
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3187
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3188
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3189
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3190
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3191
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3192
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3193
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3194
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3195
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3196
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3197
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3198
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3199
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3200
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3201
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3202
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3203
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3204
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3205
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3206
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3207
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3208
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3209
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3210
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3211
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3212
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3213
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3214
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3215
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3216
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3217
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3218
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3219
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3220
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3221
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3222
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3223
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3224
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3225
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3226
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3227
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3228
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3229
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3230
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3231
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3232
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3233
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3234
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3235
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3236
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3237
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3238
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3239
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3240
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3241
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3242
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3243
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3244
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3245
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3246
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3247
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3248
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3249
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3250
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3251
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3252
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3253
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3254
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3255
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3256
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3257
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3258
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3259
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3260
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3261
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3262
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3263
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3264
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3265
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3266
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3267
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3268
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3269
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3270
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3271
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3272
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3273
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3274
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3275
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3276
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3277
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3278
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3279
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3280
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3281
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3282
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3283
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3284
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3285
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3286
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3287
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3288
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3289
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3290
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3291
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3292
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3293
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3294
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3295
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3296
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3297
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3298
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3299
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3300
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3301
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3302
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3303
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3304
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3305
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3306
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3307
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3308
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3309
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3310
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3311
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3312
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3313
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3314
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3315
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3316
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3317
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3318
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3319
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3320
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3321
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3322
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3323
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3324
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3325
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3326
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3327
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3328
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3329
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3330
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3331
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3332
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3333
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3334
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3335
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3336
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3337
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3338
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3339
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3340
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3341
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3342
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3343
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3344
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3345
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3346
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3347
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3348
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3349
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3350
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3351
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3352
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3353
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3354
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3355
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3356
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3357
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3358
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3359
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3360
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3361
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3362
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3363
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3364
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3365
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3366
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3367
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3368
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3369
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3370
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3371
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3372
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3373
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3374
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3375
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3376
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3377
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3378
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3379
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3380
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3381
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3382
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3383
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3384
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3385
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3386
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3387
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3388
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3389
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3390
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3391
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3392
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3393
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3394
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3395
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3396
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3397
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3398
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3399
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3400
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3401
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3402
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3403
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3404
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3405
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3406
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3407
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3408
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3409
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3410
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3411
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3412
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3413
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3414
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3415
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3416
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3417
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3418
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3419
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3420
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3421
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3422
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3423
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3424
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3425
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3426
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3427
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3428
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3429
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3430
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3431
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3432
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3433
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3434
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3435
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3436
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3437
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3438
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3439
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3440
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3441
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3442
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3443
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3444
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3445
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3446
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3447
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3448
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3449
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3450
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3451
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3452
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3453
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3454
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3455
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3456
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3457
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3458
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3459
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3460
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3461
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3462
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3463
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3464
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3465
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3466
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3467
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3468
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3469
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3470
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3471
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3472
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3473
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3474
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3475
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3476
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3477
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3478
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3479
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3480
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3481
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3482
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3483
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3484
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3485
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3486
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3487
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3488
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3489
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3490
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3491
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3492
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3493
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3494
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3495
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3496
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3497
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3498
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3499
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3500
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3501
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3502
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3503
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3504
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3505
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3506
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3507
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3508
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3509
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3510
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3511
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3512
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3513
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3514
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3515
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3516
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3517
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3518
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3519
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3520
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3521
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3522
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3523
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3524
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3525
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3526
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3527
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3528
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3529
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3530
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3531
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3532
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3533
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3534
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3535
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3536
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3537
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3538
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3539
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3540
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3541
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3542
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3543
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3544
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3545
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3546
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3547
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3548
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3549
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3550
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3551
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3552
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3553
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3554
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3555
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3556
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3557
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3558
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3559
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3560
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3561
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3562
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3563
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3564
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3565
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3566
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3567
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3568
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3569
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3570
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3571
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3572
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3573
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3574
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3575
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3576
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3577
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3578
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3579
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3580
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3581
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3582
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3583
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3584
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3585
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3586
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3587
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3588
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3589
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3590
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3591
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3592
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3593
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3594
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3595
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3596
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3597
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3598
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3599
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3600
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3601
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3602
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3603
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3604
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3605
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3606
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3607
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3608
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3609
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3610
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3611
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3612
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3613
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3614
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3615
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3616
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3617
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3618
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3619
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3620
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3621
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3622
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3623
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3624
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3625
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3626
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3627
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3628
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3629
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3630
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3631
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3632
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3633
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3634
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3635
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3636
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3637
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3638
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3639
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3640
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3641
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3642
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3643
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3644
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3645
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3646
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3647
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3648
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3649
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3650
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3651
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3652
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3653
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3654
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3655
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3656
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3657
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3658
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3659
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3660
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3661
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3662
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3663
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3664
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3665
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3666
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3667
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3668
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3669
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3670
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3671
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3672
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3673
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3674
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3675
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3676
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3677
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3678
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3679
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3680
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3681
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3682
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3683
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3684
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3685
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3686
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3687
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3688
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3689
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3690
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3691
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3692
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3693
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3694
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3695
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3696
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3697
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3698
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3699
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3700
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3701
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3702
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3703
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3704
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3705
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3706
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3707
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3708
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3709
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3710
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3711
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3712
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3713
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3714
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3715
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3716
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3717
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3718
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3719
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3720
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3721
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3722
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3723
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3724
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3725
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3726
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3727
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3728
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3729
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3730
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3731
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3732
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3733
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3734
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3735
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3736
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3737
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3738
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3739
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3740
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3741
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3742
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3743
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3744
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3745
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3746
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3747
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3748
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3749
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3750
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3751
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3752
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3753
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3754
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3755
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3756
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3757
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3758
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3759
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3760
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3761
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3762
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3763
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3764
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3765
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3766
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3767
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3768
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3769
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3770
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3771
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3772
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3773
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3774
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3775
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3776
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3777
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3778
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3779
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3780
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3781
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3782
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3783
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3784
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3785
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3786
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3787
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3788
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3789
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3790
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3791
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3792
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3793
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3794
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3795
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3796
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3797
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3798
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3799
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3800
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3801
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3802
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3803
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3804
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3805
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3806
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3807
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3808
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3809
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3810
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3811
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3812
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3813
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3814
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3815
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3816
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3817
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3818
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3819
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3820
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3821
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3822
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3823
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3824
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3825
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3826
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3827
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3828
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3829
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3830
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3831
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3832
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3833
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3834
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3835
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3836
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3837
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3838
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3839
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3840
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3841
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3842
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3843
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3844
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3845
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3846
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3847
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3848
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3849
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3850
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3851
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3852
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3853
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3854
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3855
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3856
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3857
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3858
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3859
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3860
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3861
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3862
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3863
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3864
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3865
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3866
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3867
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3868
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3869
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3870
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3871
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3872
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3873
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3874
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3875
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3876
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3877
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3878
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3879
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3880
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3881
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3882
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3883
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3884
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3885
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3886
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3887
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3888
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3889
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3890
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3891
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3892
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3893
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3894
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3895
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3896
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3897
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3898
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3899
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3900
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3901
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3902
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3903
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3904
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3905
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3906
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3907
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3908
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3909
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3910
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3911
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3912
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3913
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3914
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3915
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3916
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3917
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3918
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3919
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3920
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3921
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3922
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3923
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3924
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3925
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3926
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3927
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3928
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3929
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3930
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3931
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3932
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3933
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3934
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3935
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3936
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3937
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3938
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3939
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3940
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3941
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3942
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3943
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3944
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3945
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3946
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3947
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3948
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3949
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3950
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3951
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3952
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3953
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3954
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3955
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3956
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3957
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3958
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3959
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3960
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3961
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3962
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3963
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3964
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3965
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3966
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3967
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3968
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3969
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3970
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3971
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3972
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3973
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3974
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3975
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3976
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3977
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3978
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3979
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3980
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3981
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3982
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3983
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3984
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3985
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3986
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3987
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3988
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3989
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3990
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3991
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3992
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3993
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3994
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3995
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3996
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3997
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3998
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 3999
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4000
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4001
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4002
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4003
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4004
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4005
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4006
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4007
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4008
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4009
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4010
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4011
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4012
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4013
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4014
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4015
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4016
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4017
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4018
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4019
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4020
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4021
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4022
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4023
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4024
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4025
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4026
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4027
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4028
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4029
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4030
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4031
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4032
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4033
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4034
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4035
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4036
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4037
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4038
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4039
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4040
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4041
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4042
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4043
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4044
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4045
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4046
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4047
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4048
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4049
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4050
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4051
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4052
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4053
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4054
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4055
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4056
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4057
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4058
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4059
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4060
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4061
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4062
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4063
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4064
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4065
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4066
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4067
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4068
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4069
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4070
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4071
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4072
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4073
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4074
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4075
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4076
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4077
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4078
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4079
## Warning in FUN(X[[i]], ...): Numerical 0 probability for all classes with
## observation 4080
# Create confusion matrix to assess model fit/performance on test data
nb_tda_kde_5.50.5_n4_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
## Warning in levels(reference) != levels(data): longer object length is not a
## multiple of shorter object length
## Warning in confusionMatrix.default(data = pred0,
## as.factor(Dry_Bean_DatasetTest$Class)): Levels are not in the same order for
## reference and data. Refactoring data to match.
nb_tda_kde_5.50.5_n4_db_nb_cf0
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   81      330     1   600  314
##   BOMBAY          1    156    0        0     0     8    0
##   CALI           18      0  386        0     8     0    1
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ          13      0   22      733   569     0  475
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3615          
##                  95% CI : (0.3468, 0.3765)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2771          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000     0.78937          0.0000
## Specificity                  0.64007       0.99771     0.99248          1.0000
## Pos Pred Value               0.21538       0.94545     0.93462             NaN
## Neg Pred Value               0.98661       1.00000     0.97191          0.7395
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.08922       0.03824     0.09461          0.0000
## Detection Prevalence         0.41422       0.04044     0.10123          0.0000
## Balanced Accuracy            0.77963       0.99885     0.89092          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9844        0.000      0.0000
## Specificity                0.6451        1.000      1.0000
## Pos Pred Value             0.3140          NaN         NaN
## Neg Pred Value             0.9960        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1395        0.000      0.0000
## Detection Prevalence       0.4441        0.000      0.0000
## Balanced Accuracy          0.8147        0.500      0.5000
nb_tda_kde_5.50.5_n4_db_nb_cf0 
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction BARBUNYA BOMBAY CALI DERMASON HOROZ SEKER SIRA
##   BARBUNYA      364      0   81      330     1   600  314
##   BOMBAY          1    156    0        0     0     8    0
##   CALI           18      0  386        0     8     0    1
##   DERMASON        0      0    0        0     0     0    0
##   HOROZ          13      0   22      733   569     0  475
##   SEKER           0      0    0        0     0     0    0
##   SIRA            0      0    0        0     0     0    0
## 
## Overall Statistics
##                                           
##                Accuracy : 0.3615          
##                  95% CI : (0.3468, 0.3765)
##     No Information Rate : 0.2605          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.2771          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: BARBUNYA Class: BOMBAY Class: CALI Class: DERMASON
## Sensitivity                  0.91919       1.00000     0.78937          0.0000
## Specificity                  0.64007       0.99771     0.99248          1.0000
## Pos Pred Value               0.21538       0.94545     0.93462             NaN
## Neg Pred Value               0.98661       1.00000     0.97191          0.7395
## Prevalence                   0.09706       0.03824     0.11985          0.2605
## Detection Rate               0.08922       0.03824     0.09461          0.0000
## Detection Prevalence         0.41422       0.04044     0.10123          0.0000
## Balanced Accuracy            0.77963       0.99885     0.89092          0.5000
##                      Class: HOROZ Class: SEKER Class: SIRA
## Sensitivity                0.9844        0.000      0.0000
## Specificity                0.6451        1.000      1.0000
## Pos Pred Value             0.3140          NaN         NaN
## Neg Pred Value             0.9960        0.851      0.8064
## Prevalence                 0.1417        0.149      0.1936
## Detection Rate             0.1395        0.000      0.0000
## Detection Prevalence       0.4441        0.000      0.0000
## Balanced Accuracy          0.8147        0.500      0.5000
nb_tda_kde_5.50.5_n4_db_nb_cf0$overall
##       Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull 
##   3.615196e-01   2.770842e-01   3.467586e-01   3.764792e-01   2.605392e-01 
## AccuracyPValue  McnemarPValue 
##   6.574974e-46            NaN
nb_tda_kde_5.50.5_n4_db_nb_cf0_ov_acc<-nb_tda_kde_5.50.5_n4_db_nb_cf0$overall[1]
nb_tda_kde_5.50.5_n4_db_nb_cf0$byClass
##                 Sensitivity Specificity Pos Pred Value Neg Pred Value Precision
## Class: BARBUNYA   0.9191919   0.6400651      0.2153846      0.9866109 0.2153846
## Class: BOMBAY     1.0000000   0.9977064      0.9454545      1.0000000 0.9454545
## Class: CALI       0.7893661   0.9924812      0.9346247      0.9719116 0.9346247
## Class: DERMASON   0.0000000   1.0000000            NaN      0.7394608        NA
## Class: HOROZ      0.9844291   0.6450600      0.3140177      0.9960317 0.3140177
## Class: SEKER      0.0000000   1.0000000            NaN      0.8509804        NA
## Class: SIRA       0.0000000   1.0000000            NaN      0.8063725        NA
##                    Recall        F1 Prevalence Detection Rate
## Class: BARBUNYA 0.9191919 0.3489933 0.09705882     0.08921569
## Class: BOMBAY   1.0000000 0.9719626 0.03823529     0.03823529
## Class: CALI     0.7893661 0.8558758 0.11985294     0.09460784
## Class: DERMASON 0.0000000        NA 0.26053922     0.00000000
## Class: HOROZ    0.9844291 0.4761506 0.14166667     0.13946078
## Class: SEKER    0.0000000        NA 0.14901961     0.00000000
## Class: SIRA     0.0000000        NA 0.19362745     0.00000000
##                 Detection Prevalence Balanced Accuracy
## Class: BARBUNYA           0.41421569         0.7796285
## Class: BOMBAY             0.04044118         0.9988532
## Class: CALI               0.10122549         0.8909236
## Class: DERMASON           0.00000000         0.5000000
## Class: HOROZ              0.44411765         0.8147445
## Class: SEKER              0.00000000         0.5000000
## Class: SIRA               0.00000000         0.5000000
nb_tda_kde_5.50.5_n4_db_nb_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n4_db_nb_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

diff_drybean_tda_kde_5.50.5_nb_n4_3_fold<-(db_nb_fit_re - nb_tda_kde_5.50.5_n4_nb_fit_re)
diff_drybean_tda_kde_5.50.5_nb_n4_3_fold
##      Accuracy
## 1 -0.06547372
## 2 -0.06807403
## 3 -0.03050228
## Bayesian Tests 3-fold diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nb.n4_3_fold<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n4_3_fold),-0.01,0.01)
bst_tda_kde_5.50.5_nb.n4_3_fold
## $probLeft
## [1] 0.75
## 
## $probRope
## [1] 0.25
## 
## $probRight
## [1] 0
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nb.n4_3_fold_odds.left<-bst_tda_kde_5.50.5_nb.n4_3_fold$probLeft/bst_tda_kde_5.50.5_nb.n4_3_fold$probRight
bst_tda_kde_5.50.5_nb.n4_3_fold_odds.left
## [1] Inf
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nb.n4_3_fold<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n4_3_fold),-0.01,0.01)
bsr_tda_kde_5.50.5_nb.n4_3_fold
## $winLeft
## [1] 0.9918
## 
## $winRope
## [1] 0.0082
## 
## $winRight
## [1] 0
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nb.n4_3_fold<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n4_3_fold),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nb.n4_3_fold
## $left
## [1] 0.9571996
## 
## $rope
## [1] 0.02094023
## 
## $right
## [1] 0.02186021
# Rope Plot
plot(rope(diff_drybean_tda_kde_5.50.5_nb_n4_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n4_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb_n4_3_fold))
#bf_tda_kde_5.50.5_nb.n4_3_fold

#t_test
t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n4_3_fold))
## 
##  One Sample t-test
## 
## data:  as.matrix(diff_drybean_tda_kde_5.50.5_nb_n4_3_fold)
## t = -4.5141, df = 2, p-value = 0.04573
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.106804863 -0.002561819
## sample estimates:
##   mean of x 
## -0.05468334
### Test set diff
diff_drybean_tda_kde_5.50.5_nb.n4_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n4_db_nb_cf0_ov_acc)
diff_drybean_tda_kde_5.50.5_nb.n4_test
##  Accuracy 
## 0.5654412
## Bayesian Tests Test set diff

# Bayesian Sign Test

bst_tda_kde_5.50.5_nb.n4_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n4_test),-0.01,0.01)
bst_tda_kde_5.50.5_nb.n4_test
## $probLeft
## [1] 0
## 
## $probRope
## [1] 0.5
## 
## $probRight
## [1] 0.5
# Odds Left Bayesian Sign Test 

bst_tda_kde_5.50.5_nb.n4_test_odds.left<-bst_tda_kde_5.50.5_nb.n4_test$probLeft/bst_tda_kde_5.50.5_nb.n4_test$probRight
bst_tda_kde_5.50.5_nb.n4_test_odds.left
## [1] 0
# Bayesian Signed Rank Test

bsr_tda_kde_5.50.5_nb.n4_test<-BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n4_test),-0.01,0.01)
bsr_tda_kde_5.50.5_nb.n4_test
## $winLeft
## [1] 0
## 
## $winRope
## [1] 0.159
## 
## $winRight
## [1] 0.841
# Bayesian Correlated Test

bct_tda_kde_5.50.5_nb.n4_test<-correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n4_test),0.1,-0.01,0.01)
bct_tda_kde_5.50.5_nb.n4_test
## $left
## [1] NA
## 
## $rope
## [1] NA
## 
## $right
## [1] NA
# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb.n4_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n4_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb.n4_test)) #bf_tda_kde_5.50.5_nb.n4_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n4_test))

##Node5

#DryBean_TDA_KDE_5.50.5_n5_NbFit0 <- train(as.factor(Class) ~ ., data = tda.m_dry_bean_dataset_5.50.5.n5.vec, 
#                method = 'nb', 
 #               trControl = fitControl,
 #               metric='Accuracy')

#DryBean_TDA_KDE_5.50.5_n5_NbFit0
#DryBean_TDA_KDE_5.50.5_n5_NbFit0$resample
#nb_tda_kde_5.50.5_n5_nb_fit_re<-DryBean_TDA_KDE_5.50.5_n5_NbFit0$resample[1]

#summary(DryBean_TDA_KDE_5.50.5_n5_NbFit0)

# Predict outcome using DryBean_TDA_KDE_5.50.5_n5_NbFit0 from training data based on testing data
#pred0 <- predict(DryBean_TDA_KDE_5.50.5_n5_NbFit0, newdata= Dry_Bean_DatasetTest)

# Create confusion matrix to assess model fit/performance on test data
#nb_tda_kde_5.50.5_n5_db_nb_cf0<-confusionMatrix(data=pred0, as.factor(Dry_Bean_DatasetTest$Class))
#nb_tda_kde_5.50.5_n5_db_nb_cf0
#nb_tda_kde_5.50.5_n5_db_nb_cf0 
#nb_tda_kde_5.50.5_n5_db_nb_cf0$overall
#nb_tda_kde_5.50.5_n5_db_nb_cf0_ov_acc<-nb_tda_kde_5.50.5_n5_db_nb_cf0$overall[1]
#nb_tda_kde_5.50.5_n5_db_nb_cf0$byClass
#nb_tda_kde_5.50.5_n5_db_nb_cf0_pre_rec_f1<-nb_tda_kde_5.50.5_n5_db_nb_cf0$byClass[5:7]

###### Conduct initial Bayesian tests of non-tda-assisted RF vs. tda-assisted RF classifiers

### 3-fold diff

#diff_drybean_tda_kde_5.50.5_nb_n5_3_fold<-(db_nb_fit_re - nb_tda_kde_5.50.5_n5_nb_fit_re)
#diff_drybean_tda_kde_5.50.5_nb_n5_3_fold

## Bayesian Tests 3-fold diff

# Bayesian Sign Test

#bst_tda_kde_5.50.5_nb.n5_3_fold<-#BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n5_3_fold),-0.01,0.01)
#bst_tda_kde_5.50.5_nb.n5_3_fold

# Odds Left Bayesian Sign Test 

#bst_tda_kde_5.50.5_nb.n5_3_fold_odds.left<-bst_tda_kde_5.50.5_nb.n5_3_fold$probLeft/#bst_tda_kde_5.50.5_nb.n5_3_fold$probRight
#bst_tda_kde_5.50.5_nb.n5_3_fold_odds.left

# Bayesian Signed Rank Test

#bsr_tda_kde_5.50.5_nb.n5_3_fold<-#BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n5_3_fold),-0.01,0.01)
#bsr_tda_kde_5.50.5_nb.n5_3_fold

# Bayesian Correlated Test

#bct_tda_kde_5.50.5_nb.n5_3_fold<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n5_3_fold),0.1,-0.01,0.01)
#bct_tda_kde_5.50.5_nb.n5_3_fold

# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb_n5_3_fold,c(-0.01,0.01)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n5_3_fold = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb_n5_3_fold))
#bf_tda_kde_5.50.5_nb.n5_3_fold

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb_n5_3_fold))


### Test set diff
#diff_drybean_tda_kde_5.50.5_nb.n5_test<-(db_svm_cf_ov_acc - nb_tda_kde_5.50.5_n5_db_nb_cf0_ov_acc)
#diff_drybean_tda_kde_5.50.5_nb.n5_test


## Bayesian Tests Test set diff

# Bayesian Sign Test

#bst_tda_kde_5.50.5_nb.n5_test<-BayesianSignTest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n5_test),-0.01,0.01)
#bst_tda_kde_5.50.5_nb.n5_test

# Odds Left Bayesian Sign Test 

#bst_tda_kde_5.50.5_nb.n5_test_odds.left<-bst_tda_kde_5.50.5_nb.n5_test$probLeft/#bst_tda_kde_5.50.5_nb.n5_test$probRight
#bst_tda_kde_5.50.5_nb.n5_test_odds.left

# Bayesian Signed Rank Test

#bsr_tda_kde_5.50.5_nb.n5_test<-#BayesianSignedRank(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n5_test),-0.01,0.01)
#bsr_tda_kde_5.50.5_nb.n5_test

# Bayesian Correlated Test

#bct_tda_kde_5.50.5_nb.n5_test<-#correlatedBayesianTtest(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n5_test),0.1,-0.01,0.01)
#bct_tda_kde_5.50.5_nb.n5_test

# Rope Plot
#plot(rope(diff_drybean_tda_kde_5.50.5_nb.n5_test)))

#BayesFactor
#bf_tda_kde_5.50.5_nb.n5_test = ttestBF(x = as.matrix(diff_drybean_tda_kde_5.50.5_nb.n5_test)) #bf_tda_kde_5.50.5_nb.n5_test

#t_test
#t.test(as.matrix(diff_drybean_tda_kde_5.50.5_nb.n5_test))